tale_ndjson/
defaults.rs

1//! System defaults and configuration constants for Tale
2//!
3//! This module provides all configuration constants, system defaults, and
4//! preset configurations for the Tale log processing tool. It consolidates
5//! what were previously separate constants.rs and production_defaults.rs
6//! modules.
7
8use std::time::Duration;
9
10/// Basic I/O and processing constants
11pub mod io {
12    use super::Duration;
13
14    /// How long we wait before flushing data to stdout when tailing.
15    pub const TAIL_FLUSH_INTERVAL: Duration = Duration::from_millis(250);
16
17    /// Flush stdout when we've written at least this many lines.
18    pub const FLUSH_LINE_COUNT: u16 = 40;
19
20    /// Default capacity for line strings.
21    pub const LINE_CAPACITY: usize = 512;
22
23    /// Buffer size for reading from stdin/files.
24    pub const READ_BUFFER_SIZE: usize = 8192;
25
26    /// Default capacity for output byte buffers.
27    pub const OUTPUT_BUFFER_CAPACITY: usize = 1024;
28
29    /// The initial chunk size to use for adaptive chunked readers.
30    pub const INITIAL_CHUNK_SIZE: usize = 32 * 1024; // 32K bytes
31}
32
33/// File processing thresholds and decision constants
34pub mod processing {
35    /// Large offset threshold - offsets above this suggest large file
36    /// operations
37    pub const LARGE_OFFSET_THRESHOLD: u64 = 10_000;
38
39    /// File size requiring chunked processing when combined with large offset
40    pub const CHUNKED_WITH_OFFSET_FILE_SIZE: u64 = 100 * 1024 * 1024; // 100MB
41
42    /// File size that always requires chunked processing regardless of offset
43    pub const ALWAYS_CHUNKED_FILE_SIZE: u64 = 1024 * 1024 * 1024; // 1GB
44
45    /// Number of chunks between strategy adaptation checks
46    pub const ADAPTATION_INTERVAL: usize = 20;
47
48    /// The traditional unix block size in bytes.
49    pub const BLOCK_SIZE: u64 = 512;
50}
51
52/// Memory management constants
53pub mod memory {
54    /// Memory limit for line buffering in negative line offset mode.
55    pub const MEMORY_LIMIT_BYTES: usize = 10 * 1024 * 1024; // 10MB
56}
57
58/// System defaults for chunked file processing and configuration management
59pub struct SystemDefaults;
60
61impl SystemDefaults {
62    /// Default memory budget as percentage of system memory
63    ///
64    /// Benchmark results show 10% provides good balance:
65    /// - Sufficient for most workloads
66    /// - Leaves headroom for system operations
67    /// - Scales appropriately with system size
68    pub const DEFAULT_MEMORY_PERCENTAGE: f64 = 10.0;
69
70    /// Minimum memory budget (absolute floor)
71    ///
72    /// Below this, performance degrades significantly
73    pub const MIN_MEMORY_BUDGET: usize = 5 * 1024 * 1024; // 5MB
74
75    /// Maximum memory budget (safety ceiling)
76    ///
77    /// Even on large systems, cap at reasonable limit
78    pub const MAX_MEMORY_BUDGET: usize = 500 * 1024 * 1024; // 500MB
79
80    /// Default chunk size for initial processing
81    ///
82    /// Based on benchmarks:
83    /// - 32KB provides good initial performance
84    /// - Small enough for quick adaptation
85    /// - Large enough for efficient I/O
86    pub const DEFAULT_CHUNK_SIZE: usize = 32 * 1024; // 32KB (block-aligned)
87
88    /// Minimum chunk size (emergency floor)
89    ///
90    /// Below this, overhead dominates performance
91    pub const MIN_CHUNK_SIZE: usize = 4 * 1024; // 4KB (one block)
92
93    /// Maximum chunk size (performance ceiling)
94    ///
95    /// Larger chunks show diminishing returns
96    pub const MAX_CHUNK_SIZE: usize = 4 * 1024 * 1024; // 4MB
97
98    /// Optimal chunk size for different file sizes
99    pub fn optimal_chunk_for_file(file_size: u64) -> usize {
100        // Use 4KB block size (standard for most modern filesystems)
101        const BLOCK_SIZE: usize = 4096;
102
103        let base_size = match file_size {
104            // Tiny files (< 100KB): Minimal chunking
105            0..=102_400 => Self::MIN_CHUNK_SIZE,
106
107            // Small files (100KB - 1MB): Small chunks
108            102_401..=1_048_576 => 8 * 1024,
109
110            // Medium files (1MB - 10MB): Standard chunks
111            1_048_577..=10_485_760 => Self::DEFAULT_CHUNK_SIZE,
112
113            // Large files (10MB - 100MB): Larger chunks
114            10_485_761..=104_857_600 => 128 * 1024,
115
116            // Very large files (100MB - 1GB): Big chunks
117            104_857_601..=1_073_741_824 => 512 * 1024,
118
119            // Huge files (> 1GB): Maximum chunks
120            _ => 1024 * 1024,
121        };
122
123        // Ensure block alignment
124        base_size.div_ceil(BLOCK_SIZE) * BLOCK_SIZE
125    }
126
127    /// Get default strategy based on context
128    pub fn default_strategy() -> &'static str {
129        // Conservative strategy proved best in benchmarks:
130        // - Only 20% slower than adaptive in best case
131        // - Much more predictable memory usage
132        // - Better handling of memory pressure
133        "conservative"
134    }
135
136    /// Should use chunked processing by default?
137    pub fn should_chunk_by_default(file_size: u64) -> bool {
138        // Use chunked processing for files > 1MB
139        // Benchmarks show minimal overhead, better memory control
140        file_size > 1_048_576
141    }
142
143    /// Default batch window for multi-file processing (ms)
144    pub const DEFAULT_BATCH_WINDOW_MS: u64 = 250;
145
146    /// Default line buffer capacity
147    pub const DEFAULT_LINE_CAPACITY: usize = 512;
148
149    /// Default output buffer capacity
150    pub const DEFAULT_OUTPUT_BUFFER_CAPACITY: usize = 4096;
151
152    /// Adaptation interval (chunks between adaptation checks)
153    pub const ADAPTATION_INTERVAL: usize = 20;
154
155    /// Memory pressure thresholds (validated through benchmarking)
156    pub const MEMORY_PRESSURE_LOW_THRESHOLD: f64 = 0.60; // < 60%: Normal
157    pub const MEMORY_PRESSURE_MODERATE_THRESHOLD: f64 = 0.85; // 60-85%: Moderate
158    pub const MEMORY_PRESSURE_HIGH_THRESHOLD: f64 = 0.95; // 85-95%: High
159    // > 95%: Critical
160
161    /// Chunk size reduction factors per pressure level
162    pub const PRESSURE_FACTOR_LOW: f64 = 1.0; // No reduction
163    pub const PRESSURE_FACTOR_MODERATE: f64 = 0.8; // 20% reduction
164    pub const PRESSURE_FACTOR_HIGH: f64 = 0.5; // 50% reduction
165    pub const PRESSURE_FACTOR_CRITICAL: f64 = 0.25; // 75% reduction
166
167    /// Emergency allocation factor
168    pub const EMERGENCY_ALLOCATION_FACTOR: f64 = 0.25; // 25% of requested size
169}
170
171/// System configuration presets
172pub enum ConfigPreset {
173    /// Optimized for small files and low memory systems
174    LowMemory,
175    /// Balanced performance and memory usage
176    Balanced,
177    /// Optimized for maximum performance
178    HighPerformance,
179    /// Optimized for memory-constrained environments
180    Conservative,
181}
182
183/// Configuration settings for each preset
184pub struct PresetSettings {
185    pub memory_percentage: f64,
186    pub max_memory_mb: usize,
187    pub default_chunk_kb: usize,
188    pub max_chunk_kb: usize,
189    pub strategy: &'static str,
190    pub force_chunked: bool,
191    pub adaptation_interval: usize,
192}
193
194impl ConfigPreset {
195    /// Get settings for this preset
196    pub fn settings(&self) -> PresetSettings {
197        match self {
198            ConfigPreset::LowMemory => PresetSettings {
199                memory_percentage: 5.0,
200                max_memory_mb: 50,
201                default_chunk_kb: 16,
202                max_chunk_kb: 256,
203                strategy: "conservative",
204                force_chunked: true,
205                adaptation_interval: 10,
206            },
207
208            ConfigPreset::Balanced => PresetSettings {
209                memory_percentage: 10.0,
210                max_memory_mb: 200,
211                default_chunk_kb: 32,
212                max_chunk_kb: 2048,
213                strategy: "conservative",
214                force_chunked: false,
215                adaptation_interval: 20,
216            },
217
218            ConfigPreset::HighPerformance => PresetSettings {
219                memory_percentage: 20.0,
220                max_memory_mb: 500,
221                default_chunk_kb: 128,
222                max_chunk_kb: 4096,
223                strategy: "adaptive",
224                force_chunked: false,
225                adaptation_interval: 30,
226            },
227
228            ConfigPreset::Conservative => PresetSettings {
229                memory_percentage: 5.0,
230                max_memory_mb: 100,
231                default_chunk_kb: 16,
232                max_chunk_kb: 512,
233                strategy: "static",
234                force_chunked: true,
235                adaptation_interval: 10,
236            },
237        }
238    }
239
240    /// Detect best preset based on system resources
241    pub fn auto_detect() -> Self {
242        if let Some(memory_stats) = memory_stats::memory_stats() {
243            let total_memory_mb = memory_stats.physical_mem / (1024 * 1024);
244
245            match total_memory_mb {
246                // Very low memory systems (< 2GB)
247                0..=2048 => ConfigPreset::LowMemory,
248
249                // Low-mid memory systems (2-8GB)
250                2049..=8192 => ConfigPreset::Conservative,
251
252                // Standard systems (8-16GB)
253                8193..=16384 => ConfigPreset::Balanced,
254
255                // High memory systems (> 16GB)
256                _ => ConfigPreset::HighPerformance,
257            }
258        } else {
259            // Fallback to balanced if we can't detect
260            ConfigPreset::Balanced
261        }
262    }
263}
264
265/// Get production configuration based on environment or auto-detection
266pub fn get_system_config() -> PresetSettings {
267    // Check for environment override
268    if let Ok(preset_name) = std::env::var("TALE_PRESET") {
269        let preset = match preset_name.to_lowercase().as_str() {
270            "low" | "lowmemory" | "low_memory" => ConfigPreset::LowMemory,
271            "balanced" | "balance" => ConfigPreset::Balanced,
272            "high" | "highperformance" | "high_performance" => ConfigPreset::HighPerformance,
273            "conservative" | "conserve" => ConfigPreset::Conservative,
274            _ => {
275                eprintln!("Warning: Unknown TALE_PRESET '{}', using auto-detection", preset_name);
276                ConfigPreset::auto_detect()
277            }
278        };
279        return preset.settings();
280    }
281
282    // Auto-detect based on system
283    ConfigPreset::auto_detect().settings()
284}
285
286#[cfg(test)]
287mod tests {
288    use super::*;
289
290    #[test]
291    fn chunk_sizes_are_block_aligned() {
292        const BLOCK_SIZE: usize = 4096;
293
294        let tiny = SystemDefaults::optimal_chunk_for_file(50_000);
295        assert_eq!(tiny % BLOCK_SIZE, 0);
296        assert_eq!(tiny, SystemDefaults::MIN_CHUNK_SIZE);
297
298        let small = SystemDefaults::optimal_chunk_for_file(500_000);
299        assert_eq!(small % BLOCK_SIZE, 0);
300        assert_eq!(small, 8 * 1024);
301
302        let medium = SystemDefaults::optimal_chunk_for_file(5_000_000);
303        assert_eq!(medium % BLOCK_SIZE, 0);
304        assert_eq!(medium, SystemDefaults::DEFAULT_CHUNK_SIZE);
305
306        let large = SystemDefaults::optimal_chunk_for_file(50_000_000);
307        assert_eq!(large % BLOCK_SIZE, 0);
308        assert_eq!(large, 128 * 1024);
309
310        let huge = SystemDefaults::optimal_chunk_for_file(2_000_000_000);
311        assert_eq!(huge % BLOCK_SIZE, 0);
312        assert_eq!(huge, 1024 * 1024);
313    }
314
315    #[test]
316    fn chunking_decisions_are_good() {
317        // Should not chunk tiny files
318        assert!(!SystemDefaults::should_chunk_by_default(100_000));
319
320        // Should chunk files > 1MB
321        assert!(SystemDefaults::should_chunk_by_default(2_000_000));
322    }
323
324    #[test]
325    fn presets_are_as_expected() {
326        let low_mem = ConfigPreset::LowMemory.settings();
327        assert_eq!(low_mem.memory_percentage, 5.0);
328        assert_eq!(low_mem.strategy, "conservative");
329        assert!(low_mem.force_chunked);
330
331        let balanced = ConfigPreset::Balanced.settings();
332        assert_eq!(balanced.memory_percentage, 10.0);
333        assert_eq!(balanced.strategy, "conservative");
334        assert!(!balanced.force_chunked);
335
336        let high_perf = ConfigPreset::HighPerformance.settings();
337        assert_eq!(high_perf.memory_percentage, 20.0);
338        assert_eq!(high_perf.strategy, "adaptive");
339        assert!(!high_perf.force_chunked);
340    }
341
342    #[test]
343    fn we_have_processing_constants() {
344        use super::processing::*;
345
346        // Verify constants are reasonable
347        assert_eq!(LARGE_OFFSET_THRESHOLD, 10_000);
348        assert_eq!(CHUNKED_WITH_OFFSET_FILE_SIZE, 100 * 1024 * 1024);
349        assert_eq!(ALWAYS_CHUNKED_FILE_SIZE, 1024 * 1024 * 1024);
350        assert_eq!(ADAPTATION_INTERVAL, 20);
351    }
352
353    #[test]
354    fn we_have_io_constants() {
355        use super::io::*;
356
357        // Verify reasonable defaults
358        assert_eq!(TAIL_FLUSH_INTERVAL.as_millis(), 250);
359        assert_eq!(FLUSH_LINE_COUNT, 40);
360        assert_eq!(READ_BUFFER_SIZE, 8192);
361        assert_eq!(INITIAL_CHUNK_SIZE, 32 * 1024);
362    }
363}
tale_ndjson/defaults.rs

tale_ndjson/
defaults.rs
