Skip to main content

ipfrs_storage/
quota.rs

1//! Quota Management for per-tenant storage limits
2//!
3//! This module provides quota enforcement and tracking for multi-tenant storage:
4//! - Per-tenant storage quotas
5//! - Block count limits
6//! - Bandwidth quotas (reads/writes per period)
7//! - Quota enforcement with soft/hard limits
8//! - Usage tracking and reporting
9//! - Quota alerts and notifications
10
11use crate::traits::BlockStore;
12use async_trait::async_trait;
13use dashmap::DashMap;
14use ipfrs_core::{Block, Cid, Error, Result};
15use serde::{Deserialize, Serialize};
16use std::sync::atomic::{AtomicU64, Ordering};
17use std::sync::Arc;
18use std::time::{Duration, SystemTime};
19use tracing::{debug, warn};
20
21/// Tenant identifier
22pub type TenantId = String;
23
24/// Quota configuration for a tenant
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub struct QuotaConfig {
27    /// Maximum storage in bytes (0 = unlimited)
28    pub max_bytes: u64,
29    /// Maximum number of blocks (0 = unlimited)
30    pub max_blocks: u64,
31    /// Maximum read bandwidth per period (bytes/sec, 0 = unlimited)
32    pub max_read_bandwidth: u64,
33    /// Maximum write bandwidth per period (bytes/sec, 0 = unlimited)
34    pub max_write_bandwidth: u64,
35    /// Soft limit threshold (percentage, e.g., 80 for 80%)
36    pub soft_limit_percent: u8,
37    /// Hard limit enforcement (reject on exceed)
38    pub hard_limit_enabled: bool,
39}
40
41impl Default for QuotaConfig {
42    fn default() -> Self {
43        Self {
44            max_bytes: 0,
45            max_blocks: 0,
46            max_read_bandwidth: 0,
47            max_write_bandwidth: 0,
48            soft_limit_percent: 80,
49            hard_limit_enabled: true,
50        }
51    }
52}
53
54/// Quota usage statistics
55#[derive(Debug)]
56pub struct QuotaUsage {
57    /// Current storage used in bytes
58    pub bytes_used: AtomicU64,
59    /// Current number of blocks
60    pub blocks_count: AtomicU64,
61    /// Total bytes read in current period
62    pub bytes_read: AtomicU64,
63    /// Total bytes written in current period
64    pub bytes_written: AtomicU64,
65    /// Number of quota violations
66    pub violations: AtomicU64,
67    /// Last reset time for bandwidth tracking
68    pub last_reset: parking_lot::Mutex<SystemTime>,
69}
70
71impl QuotaUsage {
72    fn new() -> Self {
73        Self {
74            bytes_used: AtomicU64::new(0),
75            blocks_count: AtomicU64::new(0),
76            bytes_read: AtomicU64::new(0),
77            bytes_written: AtomicU64::new(0),
78            violations: AtomicU64::new(0),
79            last_reset: parking_lot::Mutex::new(SystemTime::now()),
80        }
81    }
82
83    fn record_write(&self, bytes: u64) {
84        self.bytes_used.fetch_add(bytes, Ordering::Relaxed);
85        self.blocks_count.fetch_add(1, Ordering::Relaxed);
86        self.bytes_written.fetch_add(bytes, Ordering::Relaxed);
87    }
88
89    fn record_read(&self, bytes: u64) {
90        self.bytes_read.fetch_add(bytes, Ordering::Relaxed);
91    }
92
93    fn record_delete(&self, bytes: u64) {
94        self.bytes_used.fetch_sub(bytes, Ordering::Relaxed);
95        self.blocks_count.fetch_sub(1, Ordering::Relaxed);
96    }
97
98    fn record_violation(&self) {
99        self.violations.fetch_add(1, Ordering::Relaxed);
100    }
101
102    fn reset_bandwidth(&self) {
103        self.bytes_read.store(0, Ordering::Relaxed);
104        self.bytes_written.store(0, Ordering::Relaxed);
105        *self.last_reset.lock() = SystemTime::now();
106    }
107
108    fn should_reset(&self, period: Duration) -> bool {
109        let last = *self.last_reset.lock();
110        SystemTime::now().duration_since(last).unwrap_or_default() > period
111    }
112}
113
114/// Quota status
115#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
116pub enum QuotaStatus {
117    /// Within limits
118    Ok,
119    /// Exceeded soft limit (warning)
120    SoftLimitExceeded,
121    /// Exceeded hard limit (rejected)
122    HardLimitExceeded,
123}
124
125/// Quota violation type
126#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
127pub enum ViolationType {
128    /// Storage bytes exceeded
129    StorageBytes,
130    /// Block count exceeded
131    BlockCount,
132    /// Read bandwidth exceeded
133    ReadBandwidth,
134    /// Write bandwidth exceeded
135    WriteBandwidth,
136}
137
138/// Quota manager configuration
139#[derive(Debug, Clone)]
140pub struct QuotaManagerConfig {
141    /// Default quota for new tenants
142    pub default_quota: QuotaConfig,
143    /// Bandwidth tracking period
144    pub bandwidth_period: Duration,
145    /// Enable quota enforcement
146    pub enforcement_enabled: bool,
147}
148
149impl Default for QuotaManagerConfig {
150    fn default() -> Self {
151        Self {
152            default_quota: QuotaConfig::default(),
153            bandwidth_period: Duration::from_secs(60),
154            enforcement_enabled: true,
155        }
156    }
157}
158
159/// Tenant quota information
160struct TenantQuota {
161    config: parking_lot::RwLock<QuotaConfig>,
162    usage: QuotaUsage,
163}
164
165/// Quota Manager
166///
167/// Manages per-tenant storage quotas with enforcement
168pub struct QuotaManager {
169    tenants: DashMap<TenantId, TenantQuota>,
170    config: parking_lot::RwLock<QuotaManagerConfig>,
171    /// Mapping of CID to (tenant_id, size)
172    cid_map: DashMap<Cid, (TenantId, u64)>,
173}
174
175impl QuotaManager {
176    /// Create a new quota manager
177    pub fn new(config: QuotaManagerConfig) -> Self {
178        Self {
179            tenants: DashMap::new(),
180            config: parking_lot::RwLock::new(config),
181            cid_map: DashMap::new(),
182        }
183    }
184
185    /// Set quota for a tenant
186    pub fn set_quota(&self, tenant_id: TenantId, config: QuotaConfig) {
187        self.tenants
188            .entry(tenant_id.clone())
189            .and_modify(|tenant| *tenant.config.write() = config.clone())
190            .or_insert_with(|| TenantQuota {
191                config: parking_lot::RwLock::new(config),
192                usage: QuotaUsage::new(),
193            });
194        debug!("Set quota for tenant: {}", tenant_id);
195    }
196
197    /// Get quota configuration for a tenant
198    pub fn get_quota(&self, tenant_id: &str) -> Option<QuotaConfig> {
199        self.tenants
200            .get(tenant_id)
201            .map(|tenant| tenant.config.read().clone())
202    }
203
204    /// Get quota usage for a tenant
205    pub fn get_usage(&self, tenant_id: &str) -> Option<QuotaUsageSnapshot> {
206        self.tenants
207            .get(tenant_id)
208            .map(|tenant| QuotaUsageSnapshot {
209                bytes_used: tenant.usage.bytes_used.load(Ordering::Relaxed),
210                blocks_count: tenant.usage.blocks_count.load(Ordering::Relaxed),
211                bytes_read: tenant.usage.bytes_read.load(Ordering::Relaxed),
212                bytes_written: tenant.usage.bytes_written.load(Ordering::Relaxed),
213                violations: tenant.usage.violations.load(Ordering::Relaxed),
214            })
215    }
216
217    /// Check if a write operation is allowed
218    pub fn check_write_quota(
219        &self,
220        tenant_id: &str,
221        data_size: u64,
222    ) -> std::result::Result<QuotaStatus, ViolationType> {
223        let (enforcement_enabled, bandwidth_period) = {
224            let config_guard = self.config.read();
225            (
226                config_guard.enforcement_enabled,
227                config_guard.bandwidth_period,
228            )
229        };
230
231        if !enforcement_enabled {
232            return Ok(QuotaStatus::Ok);
233        }
234
235        let tenant = match self.tenants.get(tenant_id) {
236            Some(t) => t,
237            None => {
238                // Create tenant with default quota
239                let default_quota = self.config.read().default_quota.clone();
240                self.set_quota(tenant_id.to_string(), default_quota);
241                self.tenants
242                    .get(tenant_id)
243                    .expect("tenant just inserted via set_quota")
244            }
245        };
246
247        let quota_config = tenant.config.read();
248        let usage = &tenant.usage;
249
250        // Reset bandwidth if period expired
251        if usage.should_reset(bandwidth_period) {
252            usage.reset_bandwidth();
253        }
254
255        // Check storage bytes
256        if quota_config.max_bytes > 0 {
257            let current = usage.bytes_used.load(Ordering::Relaxed);
258            let projected = current + data_size;
259            let soft_limit =
260                (quota_config.max_bytes * quota_config.soft_limit_percent as u64) / 100;
261
262            if projected > quota_config.max_bytes {
263                if quota_config.hard_limit_enabled {
264                    usage.record_violation();
265                    return Err(ViolationType::StorageBytes);
266                }
267                return Ok(QuotaStatus::HardLimitExceeded);
268            } else if projected > soft_limit {
269                warn!(
270                    "Tenant {} exceeded soft storage limit: {} / {}",
271                    tenant_id, projected, quota_config.max_bytes
272                );
273                return Ok(QuotaStatus::SoftLimitExceeded);
274            }
275        }
276
277        // Check block count
278        if quota_config.max_blocks > 0 {
279            let current = usage.blocks_count.load(Ordering::Relaxed);
280            let soft_limit =
281                (quota_config.max_blocks * quota_config.soft_limit_percent as u64) / 100;
282
283            if current + 1 > quota_config.max_blocks {
284                if quota_config.hard_limit_enabled {
285                    usage.record_violation();
286                    return Err(ViolationType::BlockCount);
287                }
288                return Ok(QuotaStatus::HardLimitExceeded);
289            } else if current + 1 > soft_limit {
290                return Ok(QuotaStatus::SoftLimitExceeded);
291            }
292        }
293
294        // Check write bandwidth
295        if quota_config.max_write_bandwidth > 0 {
296            let current = usage.bytes_written.load(Ordering::Relaxed);
297            if current + data_size > quota_config.max_write_bandwidth {
298                if quota_config.hard_limit_enabled {
299                    usage.record_violation();
300                    return Err(ViolationType::WriteBandwidth);
301                }
302                return Ok(QuotaStatus::HardLimitExceeded);
303            }
304        }
305
306        Ok(QuotaStatus::Ok)
307    }
308
309    /// Check if a read operation is allowed
310    pub fn check_read_quota(
311        &self,
312        tenant_id: &str,
313        data_size: u64,
314    ) -> std::result::Result<QuotaStatus, ViolationType> {
315        let (enforcement_enabled, bandwidth_period) = {
316            let config_guard = self.config.read();
317            (
318                config_guard.enforcement_enabled,
319                config_guard.bandwidth_period,
320            )
321        };
322
323        if !enforcement_enabled {
324            return Ok(QuotaStatus::Ok);
325        }
326
327        let tenant = match self.tenants.get(tenant_id) {
328            Some(t) => t,
329            None => return Ok(QuotaStatus::Ok), // Allow reads for unknown tenants
330        };
331
332        let quota_config = tenant.config.read();
333        let usage = &tenant.usage;
334
335        // Reset bandwidth if period expired
336        if usage.should_reset(bandwidth_period) {
337            usage.reset_bandwidth();
338        }
339
340        // Check read bandwidth
341        if quota_config.max_read_bandwidth > 0 {
342            let current = usage.bytes_read.load(Ordering::Relaxed);
343            if current + data_size > quota_config.max_read_bandwidth {
344                if quota_config.hard_limit_enabled {
345                    usage.record_violation();
346                    return Err(ViolationType::ReadBandwidth);
347                }
348                return Ok(QuotaStatus::HardLimitExceeded);
349            }
350        }
351
352        Ok(QuotaStatus::Ok)
353    }
354
355    /// Record a write operation
356    pub fn record_write(&self, tenant_id: &str, cid: Cid, data_size: u64) {
357        if let Some(tenant) = self.tenants.get(tenant_id) {
358            tenant.usage.record_write(data_size);
359            self.cid_map.insert(cid, (tenant_id.to_string(), data_size));
360        }
361    }
362
363    /// Record a read operation
364    pub fn record_read(&self, tenant_id: &str, data_size: u64) {
365        if let Some(tenant) = self.tenants.get(tenant_id) {
366            tenant.usage.record_read(data_size);
367        }
368    }
369
370    /// Record a delete operation
371    pub fn record_delete(&self, cid: &Cid) {
372        if let Some((_, (tenant_id, data_size))) = self.cid_map.remove(cid) {
373            if let Some(tenant) = self.tenants.get(&tenant_id) {
374                tenant.usage.record_delete(data_size);
375            }
376        }
377    }
378
379    /// Get all tenants
380    pub fn list_tenants(&self) -> Vec<TenantId> {
381        self.tenants
382            .iter()
383            .map(|entry| entry.key().clone())
384            .collect()
385    }
386
387    /// Get quota report for a tenant
388    pub fn get_quota_report(&self, tenant_id: &str) -> Option<QuotaReport> {
389        let tenant = self.tenants.get(tenant_id)?;
390        let config = tenant.config.read().clone();
391        let usage_snapshot = QuotaUsageSnapshot {
392            bytes_used: tenant.usage.bytes_used.load(Ordering::Relaxed),
393            blocks_count: tenant.usage.blocks_count.load(Ordering::Relaxed),
394            bytes_read: tenant.usage.bytes_read.load(Ordering::Relaxed),
395            bytes_written: tenant.usage.bytes_written.load(Ordering::Relaxed),
396            violations: tenant.usage.violations.load(Ordering::Relaxed),
397        };
398
399        let storage_percent = if config.max_bytes > 0 {
400            usage_snapshot.bytes_used as f64 / config.max_bytes as f64 * 100.0
401        } else {
402            0.0
403        };
404
405        let blocks_percent = if config.max_blocks > 0 {
406            usage_snapshot.blocks_count as f64 / config.max_blocks as f64 * 100.0
407        } else {
408            0.0
409        };
410
411        Some(QuotaReport {
412            tenant_id: tenant_id.to_string(),
413            config,
414            usage: usage_snapshot,
415            storage_utilization_percent: storage_percent,
416            blocks_utilization_percent: blocks_percent,
417        })
418    }
419}
420
421/// Snapshot of quota usage
422#[derive(Debug, Clone, Serialize, Deserialize)]
423pub struct QuotaUsageSnapshot {
424    pub bytes_used: u64,
425    pub blocks_count: u64,
426    pub bytes_read: u64,
427    pub bytes_written: u64,
428    pub violations: u64,
429}
430
431/// Quota report
432#[derive(Debug, Clone, Serialize, Deserialize)]
433pub struct QuotaReport {
434    pub tenant_id: TenantId,
435    pub config: QuotaConfig,
436    pub usage: QuotaUsageSnapshot,
437    pub storage_utilization_percent: f64,
438    pub blocks_utilization_percent: f64,
439}
440
441/// Quota-enforced block store
442pub struct QuotaBlockStore<S: BlockStore> {
443    inner: Arc<S>,
444    quota_manager: Arc<QuotaManager>,
445    tenant_id: TenantId,
446}
447
448impl<S: BlockStore> QuotaBlockStore<S> {
449    /// Create a new quota-enforced block store
450    pub fn new(inner: Arc<S>, quota_manager: Arc<QuotaManager>, tenant_id: TenantId) -> Self {
451        Self {
452            inner,
453            quota_manager,
454            tenant_id,
455        }
456    }
457
458    /// Get the quota manager
459    pub fn quota_manager(&self) -> &Arc<QuotaManager> {
460        &self.quota_manager
461    }
462
463    /// Get the tenant ID
464    pub fn tenant_id(&self) -> &str {
465        &self.tenant_id
466    }
467}
468
469#[async_trait]
470impl<S: BlockStore + Send + Sync + 'static> BlockStore for QuotaBlockStore<S> {
471    async fn get(&self, cid: &Cid) -> Result<Option<Block>> {
472        let block_opt = self.inner.get(cid).await?;
473
474        if let Some(ref block) = block_opt {
475            // Check read quota
476            match self
477                .quota_manager
478                .check_read_quota(&self.tenant_id, block.data().len() as u64)
479            {
480                Ok(QuotaStatus::Ok) => {
481                    self.quota_manager
482                        .record_read(&self.tenant_id, block.data().len() as u64);
483                }
484                Ok(QuotaStatus::SoftLimitExceeded) => {
485                    warn!("Tenant {} exceeded soft read quota limit", self.tenant_id);
486                    self.quota_manager
487                        .record_read(&self.tenant_id, block.data().len() as u64);
488                }
489                Ok(QuotaStatus::HardLimitExceeded) | Err(_) => {
490                    return Err(Error::InvalidInput(format!(
491                        "Tenant {} exceeded read quota",
492                        self.tenant_id
493                    )))
494                }
495            }
496        }
497
498        Ok(block_opt)
499    }
500
501    async fn put(&self, block: &Block) -> Result<()> {
502        // Check write quota before writing
503        match self
504            .quota_manager
505            .check_write_quota(&self.tenant_id, block.data().len() as u64)
506        {
507            Ok(QuotaStatus::Ok) => {}
508            Ok(QuotaStatus::SoftLimitExceeded) => {
509                warn!(
510                    "Tenant {} exceeded soft storage quota limit",
511                    self.tenant_id
512                );
513            }
514            Ok(QuotaStatus::HardLimitExceeded) | Err(_) => {
515                return Err(Error::InvalidInput(format!(
516                    "Tenant {} exceeded storage quota",
517                    self.tenant_id
518                )))
519            }
520        }
521
522        self.inner.put(block).await?;
523        self.quota_manager
524            .record_write(&self.tenant_id, *block.cid(), block.data().len() as u64);
525        Ok(())
526    }
527
528    async fn has(&self, cid: &Cid) -> Result<bool> {
529        self.inner.has(cid).await
530    }
531
532    async fn delete(&self, cid: &Cid) -> Result<()> {
533        self.inner.delete(cid).await?;
534        self.quota_manager.record_delete(cid);
535        Ok(())
536    }
537
538    fn list_cids(&self) -> Result<Vec<Cid>> {
539        self.inner.list_cids()
540    }
541
542    fn len(&self) -> usize {
543        self.inner.len()
544    }
545}
546
547#[cfg(test)]
548mod tests {
549    use super::*;
550    use crate::memory::MemoryBlockStore;
551    use bytes::Bytes;
552
553    #[tokio::test]
554    async fn test_quota_enforcement() {
555        let config = QuotaManagerConfig {
556            default_quota: QuotaConfig {
557                max_bytes: 1000,
558                max_blocks: 10,
559                hard_limit_enabled: true,
560                ..Default::default()
561            },
562            enforcement_enabled: true,
563            ..Default::default()
564        };
565
566        let manager = Arc::new(QuotaManager::new(config));
567        let store = Arc::new(MemoryBlockStore::new());
568        let quota_store = QuotaBlockStore::new(store, manager.clone(), "tenant1".to_string());
569
570        let data = vec![0u8; 100];
571        let block = Block::new(Bytes::from(data)).unwrap();
572
573        // Should succeed
574        quota_store.put(&block).await.unwrap();
575
576        // Check usage
577        let usage = manager.get_usage("tenant1").unwrap();
578        assert_eq!(usage.bytes_used, 100);
579        assert_eq!(usage.blocks_count, 1);
580    }
581
582    #[tokio::test]
583    async fn test_quota_exceeded() {
584        let config = QuotaManagerConfig {
585            default_quota: QuotaConfig {
586                max_bytes: 50,
587                hard_limit_enabled: true,
588                ..Default::default()
589            },
590            enforcement_enabled: true,
591            ..Default::default()
592        };
593
594        let manager = Arc::new(QuotaManager::new(config));
595        let store = Arc::new(MemoryBlockStore::new());
596        let quota_store = QuotaBlockStore::new(store, manager, "tenant1".to_string());
597
598        let data = vec![0u8; 100];
599        let block = Block::new(Bytes::from(data)).unwrap();
600
601        // Should fail (exceeds quota)
602        let result = quota_store.put(&block).await;
603        assert!(result.is_err());
604    }
605
606    #[test]
607    fn test_quota_report() {
608        let manager = QuotaManager::new(QuotaManagerConfig::default());
609        manager.set_quota(
610            "tenant1".to_string(),
611            QuotaConfig {
612                max_bytes: 1000,
613                max_blocks: 100,
614                ..Default::default()
615            },
616        );
617
618        let cid = cid::Cid::default();
619        manager.record_write("tenant1", cid, 500);
620
621        let report = manager.get_quota_report("tenant1").unwrap();
622        assert_eq!(report.usage.bytes_used, 500);
623        assert_eq!(report.storage_utilization_percent, 50.0);
624    }
625}