kawa-storage 0.1.1

High-performance storage engine for Kawa message broker
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
//! # Security Module for DoS Protection
//!
//! Rate limiting, Resource limits, GPU timeout mechanisms
//! Critical security improvements for production environment

use crate::{StorageError, StorageResult};
use std::{
    sync::{Arc, atomic::{AtomicU64, AtomicUsize, Ordering}, RwLock},
    collections::HashMap,
    time::{Duration, Instant, SystemTime},
};
use tokio::time::timeout;

/// セキュリティ設定
#[derive(Debug, Clone)]
pub struct SecurityConfig {
    /// 最大イベントサイズ(バイト)
    pub max_event_size: usize,
    /// 最大バッチサイズ(イベント数)
    pub max_batch_size: usize,
    /// 最大同時バッチ数
    pub max_concurrent_batches: usize,
    /// GPUタイムアウト(ミリ秒)
    pub gpu_timeout_ms: u64,
    /// レート制限(events/sec)
    pub rate_limit_per_second: u64,
    /// メモリ使用量制限(バイト)
    pub max_memory_usage: usize,
    /// CPU使用率制限(パーセント)
    pub max_cpu_usage: f32,
    /// GPU使用率制限(パーセント)
    pub max_gpu_usage: f32,
}

impl Default for SecurityConfig {
    fn default() -> Self {
        Self {
            max_event_size: 1024 * 1024,        // 1MB
            max_batch_size: 10_000,             // 10K events
            max_concurrent_batches: 100,        // 100 batches
            gpu_timeout_ms: 5_000,              // 5 seconds
            rate_limit_per_second: 100_000,     // 100K events/sec
            max_memory_usage: 2 * 1024 * 1024 * 1024, // 2GB
            max_cpu_usage: 80.0,                // 80%
            max_gpu_usage: 90.0,                // 90%
        }
    }
}

/// セキュリティマネージャー
pub struct SecurityManager {
    config: SecurityConfig,
    rate_limiter: Arc<RateLimiter>,
    resource_monitor: Arc<ResourceMonitor>,
    gpu_timeout_manager: Arc<GPUTimeoutManager>,
    security_metrics: SecurityMetrics,
}

/// レート制限機能
pub struct RateLimiter {
    /// 現在の秒における処理済みイベント数
    current_second_count: AtomicU64,
    /// 現在の秒のタイムスタンプ
    current_second: AtomicU64,
    /// 制限値
    limit_per_second: u64,
    /// 違反回数
    violations: AtomicU64,
}

/// リソース監視機能
pub struct ResourceMonitor {
    /// アクティブなバッチ数
    active_batches: AtomicUsize,
    /// 現在のメモリ使用量
    current_memory_usage: AtomicUsize,
    /// 現在のCPU使用率
    current_cpu_usage: AtomicU64, // f32 * 1000 for atomic
    /// 現在のGPU使用率
    current_gpu_usage: AtomicU64, // f32 * 1000 for atomic
    /// 最大値記録
    peak_memory: AtomicUsize,
    peak_cpu: AtomicU64,
    peak_gpu: AtomicU64,
}

/// GPU タイムアウト管理
pub struct GPUTimeoutManager {
    /// アクティブなGPUタスク
    active_gpu_tasks: RwLock<HashMap<u64, GPUTaskInfo>>,
    /// タスクIDカウンター
    task_id_counter: AtomicU64,
    /// タイムアウト時間
    timeout_duration: Duration,
    /// タイムアウト発生回数
    timeout_count: AtomicU64,
}

/// GPUタスク情報
#[derive(Debug, Clone)]
pub struct GPUTaskInfo {
    pub task_id: u64,
    pub start_time: Instant,
    pub event_count: usize,
    pub gpu_device: String,
}

/// セキュリティメトリクス
#[derive(Debug, Default)]
pub struct SecurityMetrics {
    /// レート制限違反回数
    pub rate_limit_violations: AtomicU64,
    /// リソース制限違反回数
    pub resource_limit_violations: AtomicU64,
    /// GPUタイムアウト回数
    pub gpu_timeouts: AtomicU64,
    /// 拒否されたリクエスト数
    pub rejected_requests: AtomicU64,
    /// セキュリティイベント数
    pub security_events: AtomicU64,
}

/// セキュリティエラー
#[derive(Debug, Clone)]
pub enum SecurityError {
    RateLimitExceeded { current: u64, limit: u64 },
    BatchTooLarge { size: usize, limit: usize },
    TooManyActiveBatches { active: usize, limit: usize },
    EventTooLarge { size: usize, limit: usize },
    MemoryExhausted { current: usize, limit: usize },
    CPUExhausted { current: f32, limit: f32 },
    GPUExhausted { current: f32, limit: f32 },
    GPUTimeout { task_id: u64, duration: Duration },
    ResourceMonitoringFailed,
}

impl SecurityManager {
    /// 新しいセキュリティマネージャーを作成
    pub fn new(config: SecurityConfig) -> Self {
        let rate_limiter = Arc::new(RateLimiter::new(config.rate_limit_per_second));
        let resource_monitor = Arc::new(ResourceMonitor::new());
        let gpu_timeout_manager = Arc::new(GPUTimeoutManager::new(
            Duration::from_millis(config.gpu_timeout_ms)
        ));
        
        tracing::info!(
            "SecurityManager initialized: rate_limit={}/sec, max_batch={}, gpu_timeout={}ms",
            config.rate_limit_per_second,
            config.max_batch_size,
            config.gpu_timeout_ms
        );
        
        Self {
            config,
            rate_limiter,
            resource_monitor,
            gpu_timeout_manager,
            security_metrics: SecurityMetrics::default(),
        }
    }
    
    /// バッチ処理前のセキュリティ検証
    pub async fn validate_batch_request(
        &self,
        event_count: usize,
        total_size: usize,
    ) -> Result<(), SecurityError> {
        // 1. レート制限チェック
        self.rate_limiter.check_rate(event_count as u64)
            .map_err(|_| {
                self.security_metrics.rate_limit_violations.fetch_add(1, Ordering::Relaxed);
                SecurityError::RateLimitExceeded {
                    current: event_count as u64,
                    limit: self.config.rate_limit_per_second,
                }
            })?;
        
        // 2. バッチサイズ検証
        if event_count > self.config.max_batch_size {
            self.security_metrics.rejected_requests.fetch_add(1, Ordering::Relaxed);
            return Err(SecurityError::BatchTooLarge {
                size: event_count,
                limit: self.config.max_batch_size,
            });
        }
        
        // 3. リソース制限チェック
        self.resource_monitor.check_resources(&self.config)
            .map_err(|e| {
                self.security_metrics.resource_limit_violations.fetch_add(1, Ordering::Relaxed);
                e
            })?;
        
        tracing::debug!(
            "Security validation passed: {} events, {} bytes",
            event_count,
            total_size
        );
        
        Ok(())
    }
    
    /// GPU処理のセキュアラッパー
    pub async fn secure_gpu_process<F, T>(
        &self,
        task_name: &str,
        event_count: usize,
        gpu_future: F,
    ) -> Result<T, SecurityError>
    where
        F: std::future::Future<Output = Result<T, crate::StorageError>>,
    {
        // GPUタスク登録
        let task_id = self.gpu_timeout_manager.register_task(
            task_name.to_string(),
            event_count,
        ).await;
        
        tracing::debug!(
            "Starting secure GPU process: task_id={}, events={}, timeout={}ms",
            task_id,
            event_count,
            self.config.gpu_timeout_ms
        );
        
        // タイムアウト付きGPU処理実行
        let result = timeout(
            Duration::from_millis(self.config.gpu_timeout_ms),
            gpu_future
        ).await;
        
        // タスク登録解除
        self.gpu_timeout_manager.unregister_task(task_id).await;
        
        match result {
            Ok(Ok(value)) => {
                tracing::debug!("GPU task completed successfully: task_id={}", task_id);
                Ok(value)
            }
            Ok(Err(_storage_err)) => {
                Err(SecurityError::ResourceMonitoringFailed)
            }
            Err(_timeout_err) => {
                self.security_metrics.gpu_timeouts.fetch_add(1, Ordering::Relaxed);
                tracing::error!(
                    "GPU task timeout: task_id={}, duration={}ms",
                    task_id,
                    self.config.gpu_timeout_ms
                );
                Err(SecurityError::GPUTimeout {
                    task_id,
                    duration: Duration::from_millis(self.config.gpu_timeout_ms),
                })
            }
        }
    }
    
    /// セキュリティメトリクス取得
    pub fn get_security_metrics(&self) -> SecurityMetrics {
        SecurityMetrics {
            rate_limit_violations: AtomicU64::new(
                self.security_metrics.rate_limit_violations.load(Ordering::Relaxed)
            ),
            resource_limit_violations: AtomicU64::new(
                self.security_metrics.resource_limit_violations.load(Ordering::Relaxed)
            ),
            gpu_timeouts: AtomicU64::new(
                self.security_metrics.gpu_timeouts.load(Ordering::Relaxed)
            ),
            rejected_requests: AtomicU64::new(
                self.security_metrics.rejected_requests.load(Ordering::Relaxed)
            ),
            security_events: AtomicU64::new(
                self.security_metrics.security_events.load(Ordering::Relaxed)
            ),
        }
    }
}

impl RateLimiter {
    /// 新しいレート制限機能を作成
    pub fn new(limit_per_second: u64) -> Self {
        Self {
            current_second_count: AtomicU64::new(0),
            current_second: AtomicU64::new(Self::current_timestamp()),
            limit_per_second,
            violations: AtomicU64::new(0),
        }
    }
    
    /// レート制限チェック
    pub fn check_rate(&self, event_count: u64) -> Result<(), ()> {
        let now = Self::current_timestamp();
        let current_sec = self.current_second.load(Ordering::Relaxed);
        
        // 秒が変わった場合はカウンターリセット
        if now != current_sec {
            if self.current_second.compare_exchange(current_sec, now, Ordering::Relaxed, Ordering::Relaxed).is_ok() {
                self.current_second_count.store(0, Ordering::Relaxed);
            }
        }
        
        // 現在の秒でのカウント増加
        let new_count = self.current_second_count.fetch_add(event_count, Ordering::Relaxed) + event_count;
        
        if new_count > self.limit_per_second {
            // カウントを元に戻す
            self.current_second_count.fetch_sub(event_count, Ordering::Relaxed);
            self.violations.fetch_add(1, Ordering::Relaxed);
            
            tracing::warn!(
                "Rate limit exceeded: current={}, limit={}, violations={}",
                new_count,
                self.limit_per_second,
                self.violations.load(Ordering::Relaxed)
            );
            
            return Err(());
        }
        
        Ok(())
    }
    
    /// 現在のタイムスタンプ(秒)を取得
    fn current_timestamp() -> u64 {
        SystemTime::now()
            .duration_since(SystemTime::UNIX_EPOCH)
            .unwrap_or_default()
            .as_secs()
    }
}

impl ResourceMonitor {
    /// 新しいリソース監視機能を作成
    pub fn new() -> Self {
        Self {
            active_batches: AtomicUsize::new(0),
            current_memory_usage: AtomicUsize::new(0),
            current_cpu_usage: AtomicU64::new(0),
            current_gpu_usage: AtomicU64::new(0),
            peak_memory: AtomicUsize::new(0),
            peak_cpu: AtomicU64::new(0),
            peak_gpu: AtomicU64::new(0),
        }
    }
    
    /// リソース制限チェック
    pub fn check_resources(&self, config: &SecurityConfig) -> Result<(), SecurityError> {
        // アクティブバッチ数チェック
        let active = self.active_batches.load(Ordering::Relaxed);
        if active >= config.max_concurrent_batches {
            return Err(SecurityError::TooManyActiveBatches {
                active,
                limit: config.max_concurrent_batches,
            });
        }
        
        // メモリ使用量チェック
        let memory = self.current_memory_usage.load(Ordering::Relaxed);
        if memory > config.max_memory_usage {
            return Err(SecurityError::MemoryExhausted {
                current: memory,
                limit: config.max_memory_usage,
            });
        }
        
        Ok(())
    }
    
    /// バッチ開始通知
    pub fn start_batch(&self) {
        self.active_batches.fetch_add(1, Ordering::Relaxed);
    }
    
    /// バッチ終了通知
    pub fn end_batch(&self) {
        self.active_batches.fetch_sub(1, Ordering::Relaxed);
    }
}

impl GPUTimeoutManager {
    /// 新しいGPUタイムアウト管理機能を作成
    pub fn new(timeout_duration: Duration) -> Self {
        Self {
            active_gpu_tasks: RwLock::new(HashMap::new()),
            task_id_counter: AtomicU64::new(0),
            timeout_duration,
            timeout_count: AtomicU64::new(0),
        }
    }
    
    /// GPUタスク登録
    pub async fn register_task(&self, gpu_device: String, event_count: usize) -> u64 {
        let task_id = self.task_id_counter.fetch_add(1, Ordering::Relaxed);
        let task_info = GPUTaskInfo {
            task_id,
            start_time: Instant::now(),
            event_count,
            gpu_device,
        };
        
        if let Ok(mut tasks) = self.active_gpu_tasks.write() {
            tasks.insert(task_id, task_info);
        }
        
        tracing::trace!("GPU task registered: task_id={}, events={}", task_id, event_count);
        task_id
    }
    
    /// GPUタスク登録解除
    pub async fn unregister_task(&self, task_id: u64) {
        if let Ok(mut tasks) = self.active_gpu_tasks.write() {
            if let Some(task_info) = tasks.remove(&task_id) {
                let duration = task_info.start_time.elapsed();
                tracing::trace!(
                    "GPU task unregistered: task_id={}, duration={:?}",
                    task_id,
                    duration
                );
            }
        }
    }
}