xet_data/deduplication/
dedup_metrics.rs

1use serde::{Deserialize, Serialize};
2
3#[derive(Default, Debug, Clone, Copy, Serialize, Deserialize)]
4#[cfg_attr(feature = "python", pyo3::pyclass(get_all, from_py_object))]
5pub struct DeduplicationMetrics {
6    pub total_bytes: u64,
7    pub deduped_bytes: u64,
8    pub new_bytes: u64,
9    pub deduped_bytes_by_global_dedup: u64,
10    pub defrag_prevented_dedup_bytes: u64,
11
12    pub total_chunks: u64,
13    pub deduped_chunks: u64,
14    pub new_chunks: u64,
15    pub deduped_chunks_by_global_dedup: u64,
16    pub defrag_prevented_dedup_chunks: u64,
17
18    pub xorb_bytes_uploaded: u64,
19    pub shard_bytes_uploaded: u64,
20    pub total_bytes_uploaded: u64,
21}
22
23/// Implement + for the metrics above, so they can be added
24/// and updated after each call to process_chunks.
25impl DeduplicationMetrics {
26    pub fn merge_in(&mut self, other: &Self) {
27        self.total_bytes += other.total_bytes;
28        self.deduped_bytes += other.deduped_bytes;
29        self.new_bytes += other.new_bytes;
30        self.deduped_bytes_by_global_dedup += other.deduped_bytes_by_global_dedup;
31        self.defrag_prevented_dedup_bytes += other.defrag_prevented_dedup_bytes;
32
33        self.total_chunks += other.total_chunks;
34        self.deduped_chunks += other.deduped_chunks;
35        self.new_chunks += other.new_chunks;
36        self.deduped_chunks_by_global_dedup += other.deduped_chunks_by_global_dedup;
37        self.defrag_prevented_dedup_chunks += other.defrag_prevented_dedup_chunks;
38
39        self.xorb_bytes_uploaded += other.xorb_bytes_uploaded;
40        self.shard_bytes_uploaded += other.shard_bytes_uploaded;
41        self.total_bytes_uploaded += other.total_bytes_uploaded;
42    }
43}
xet_data/deduplication/dedup_metrics.rs

xet_data/deduplication/
dedup_metrics.rs
