scirs2_core/memory_efficient/
mod.rs

1//! Memory-efficient operations and views for large arrays.
2//!
3//! This module provides utilities for working with large arrays efficiently by:
4//! - Using chunk-wise processing to reduce memory requirements
5//! - Creating memory-efficient views that avoid full data copies
6//! - Implementing lazy evaluation and operation fusion for improved performance
7//! - Supporting out-of-core processing for data that doesn't fit in RAM
8//! - Smart prefetching for improved performance with predictable access patterns
9
10mod adaptive_chunking;
11mod adaptive_feedback;
12mod adaptive_prefetch;
13mod chunk_format;
14mod chunked;
15#[cfg(feature = "memory_compression")]
16mod compressed_memmap;
17#[cfg(feature = "gpu")]
18mod cross_device;
19mod cross_file_prefetch;
20mod fusion;
21mod large_data;
22mod lazy_array;
23mod memmap;
24mod memmap_chunks;
25mod memmap_slice;
26mod memory_layout;
27mod numa_topology;
28mod out_of_core;
29mod out_of_core_v2;
30mod pattern_recognition;
31mod platform_memory;
32mod prefetch;
33mod resource_aware;
34#[cfg(feature = "parallel")]
35mod streaming;
36mod validation;
37mod views;
38#[cfg(feature = "parallel")]
39mod work_stealing;
40mod zero_copy_interface;
41#[cfg(feature = "parallel")]
42mod zero_copy_streaming;
43mod zero_serialization;
44mod zerocopy;
45
46pub use adaptive_chunking::{
47    create_shared_monitor,
48    // Core types
49    AdaptiveChunking,
50    AdaptiveChunkingBuilder,
51    AdaptiveChunkingParams,
52    AdaptiveChunkingResult,
53    // v0.2.0: Memory management types
54    AdaptiveChunkingWithMonitoring,
55    ChunkProcessingStats,
56    DynamicChunkAdjuster,
57    MemoryLimits,
58    MemoryPressureLevel,
59    MemoryPressureMonitor,
60    MemoryTrend,
61    SharedMemoryMonitor,
62    WorkloadType,
63};
64pub use adaptive_feedback::{
65    create_shared_predictor, ChunkSizePredictor, PerformanceMetrics, SharedPredictor,
66};
67pub use adaptive_prefetch::{
68    AdaptivePatternTracker, AdaptivePrefetchConfig, AdaptivePrefetchConfigBuilder,
69    PatternTrackerFactory, PrefetchStrategy,
70};
71pub use chunk_format::{
72    detect_format_version, read_chunk_index, read_header, write_chunk_index, write_header,
73    ChunkIndex, ChunkIndexEntry, CompressionType, FormatVersion, OutOfCoreHeaderV2,
74    HEADER_FIXED_SIZE, MAGIC_BYTES_V2,
75};
76pub use chunked::{
77    chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, ChunkedArray, ChunkingStrategy,
78    OPTIMAL_CHUNK_SIZE,
79};
80#[cfg(feature = "memory_compression")]
81pub use compressed_memmap::{
82    CompressedFileMetadata, CompressedMemMapBuilder, CompressedMemMappedArray, CompressionAlgorithm,
83};
84#[cfg(feature = "gpu")]
85pub use cross_device::{
86    create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
87    CrossDeviceManager, DeviceArray, DeviceBuffer, DeviceMemoryManager, DeviceMemoryPool,
88    DeviceStream, DeviceType, KernelParam, ToDevice, ToHost, TransferDirection, TransferEvent,
89    TransferMode, TransferOptions, TransferOptionsBuilder,
90};
91pub use cross_file_prefetch::{
92    AccessType, CrossFilePrefetchConfig, CrossFilePrefetchConfigBuilder, CrossFilePrefetchManager,
93    CrossFilePrefetchRegistry, DataAccess, DatasetId, DatasetPrefetcher,
94};
95pub use fusion::{register_fusion, FusedOp, OpFusion};
96#[cfg(feature = "parallel")]
97pub use large_data::{create_parallel_processor, ParallelStreamingProcessor};
98pub use large_data::{create_streaming_iterator, StreamingChunkIterator};
99pub use lazy_array::{evaluate, LazyArray, LazyOp, LazyOpKind};
100pub use memmap::{create_mmap, create_temp_mmap, open_mmap, AccessMode, MemoryMappedArray};
101#[cfg(feature = "parallel")]
102pub use memmap_chunks::MemoryMappedChunksParallel;
103pub use memmap_chunks::{ChunkIter, MemoryMappedChunkIter, MemoryMappedChunks};
104pub use memmap_slice::{MemoryMappedSlice, MemoryMappedSlicing};
105pub use memory_layout::{
106    AccessPattern as MemoryAccessPattern, ArrayCreation, ArrayLayout, LayoutConverter, LayoutOrder,
107    MemoryLayout,
108};
109pub use numa_topology::{NumaNode, NumaPolicy, NumaTopology};
110pub use out_of_core::{create_disk_array, load_chunks, DiskBackedArray, OutOfCoreArray};
111pub use out_of_core_v2::OutOfCoreArrayV2;
112pub use pattern_recognition::{
113    ComplexPattern, Confidence, PatternRecognitionConfig, PatternRecognizer, RecognizedPattern,
114};
115pub use platform_memory::PlatformMemoryInfo;
116#[cfg(feature = "memory_compression")]
117pub use prefetch::PrefetchingCompressedArray;
118pub use prefetch::{
119    AccessPattern, PrefetchConfig, PrefetchConfigBuilder, PrefetchStats, Prefetching,
120};
121pub use resource_aware::{
122    ResourceAwareConfig, ResourceAwareConfigBuilder, ResourceAwarePrefetcher, ResourceMonitor,
123    ResourceSnapshot, ResourceSummary, ResourceType,
124};
125#[cfg(feature = "parallel")]
126pub use streaming::{
127    create_pipeline, create_stream_processor, Pipeline, PipelineBuilder, PipelineStats,
128    StreamConfig, StreamConfigBuilder, StreamMode, StreamProcessor, StreamSource, StreamState,
129    StreamStats,
130};
131pub use views::{diagonal_view, transpose_view, view_as, view_mut_as, ArrayView, ViewMut};
132#[cfg(feature = "parallel")]
133pub use work_stealing::{
134    create_cpu_intensive_scheduler, create_io_intensive_scheduler, create_work_stealing_scheduler,
135    SchedulerStats, TaskPriority, WorkStealingConfig, WorkStealingConfigBuilder,
136    WorkStealingScheduler, WorkStealingTask,
137};
138pub use zero_copy_interface::{
139    create_global_data_registry, create_zero_copy_data, get_global_data, global_interface,
140    register_global_data, DataExchange, DataId, DataMetadata, FromZeroCopy, InterfaceStats,
141    IntoZeroCopy, ZeroCopyData, ZeroCopyInterface, ZeroCopyView, ZeroCopyWeakRef,
142};
143#[cfg(feature = "parallel")]
144pub use zero_copy_streaming::{
145    create_zero_copy_processor, BufferPool, BufferPoolStats, LockFreeQueue, ProcessingMode,
146    WorkStealingScheduler as ZeroCopyWorkStealingScheduler,
147    WorkStealingTask as ZeroCopyWorkStealingTask, ZeroCopyBuffer, ZeroCopyConfig, ZeroCopyStats,
148    ZeroCopyStreamProcessor,
149};
150pub use zero_serialization::{ZeroCopySerializable, ZeroCopySerialization};
151pub use zerocopy::{ArithmeticOps, BroadcastOps, ZeroCopyOps};
152
153// Re-export commonly used items in a prelude module for convenience
154pub mod prelude {
155    // Core functionality always available
156    pub use super::{
157        chunk_wise_binary_op, chunk_wise_op, chunk_wise_reduce, create_mmap, create_temp_mmap,
158        evaluate, open_mmap, view_as, view_mut_as, AccessMode, AdaptiveChunking,
159        AdaptiveChunkingBuilder, AdaptivePatternTracker, AdaptivePrefetchConfig, ArithmeticOps,
160        ArrayCreation, ArrayLayout, ArrayView, BroadcastOps, ChunkIter, ChunkedArray,
161        ComplexPattern, CrossFilePrefetchManager, DatasetId, DatasetPrefetcher, LayoutOrder,
162        LazyArray, MemoryLayout, MemoryMappedArray, MemoryMappedChunkIter, MemoryMappedChunks,
163        MemoryMappedSlice, MemoryMappedSlicing, OutOfCoreArray, PatternRecognizer, PrefetchConfig,
164        PrefetchConfigBuilder, PrefetchStrategy, Prefetching, ResourceAwareConfig,
165        ResourceAwarePrefetcher, ViewMut, ZeroCopyData, ZeroCopyInterface, ZeroCopyOps,
166        ZeroCopySerializable, ZeroCopySerialization, ZeroCopyView,
167    };
168
169    // GPU-specific exports
170    #[cfg(feature = "gpu")]
171    pub use super::{
172        create_cpuarray, create_cross_device_manager, create_gpuarray, to_best_device,
173    };
174
175    // Parallel processing exports
176    #[cfg(feature = "parallel")]
177    pub use super::MemoryMappedChunksParallel;
178}
scirs2_core/memory_efficient/mod.rs

scirs2_core/memory_efficient/
mod.rs
