use crate::protocol::RecordBatchReader;
use anyhow::Result;
use bytes::Bytes;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex;
pub struct LogCompactor;
impl LogCompactor {
pub fn compact_batches(data: Bytes) -> Result<Vec<crate::protocol::Record>> {
let mut reader = RecordBatchReader::new(data);
let mut latest: HashMap<Bytes, (i64, crate::protocol::Record)> = HashMap::new();
let mut no_key_records: Vec<crate::protocol::Record> = Vec::new();
let mut next_offset = 0i64;
while let Some((batch, _)) = reader.next_batch()? {
for record in batch.records {
if let Some(key) = &record.key {
latest.insert(key.clone(), (next_offset, record));
} else {
no_key_records.push(record);
}
next_offset += 1;
}
}
let mut key_records: Vec<(i64, crate::protocol::Record)> =
latest.into_values().collect();
key_records.sort_by_key(|(off, _)| *off);
let mut out: Vec<crate::protocol::Record> = Vec::new();
for (_, r) in key_records {
out.push(r);
}
out.extend(no_key_records);
Ok(out)
}
}
pub struct CompactWorker {
_inner: Arc<Mutex<()>>,
}
impl CompactWorker {
pub fn new() -> Self {
Self {
_inner: Arc::new(Mutex::new(())),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::protocol::recordbatch::{Compression, Record, RecordBatch};
use bytes::Bytes;
#[test]
fn test_compaction_keeps_latest() {
let records = vec![
Record {
attributes: 0,
timestamp_delta: 0,
offset_delta: 0,
key: Some(Bytes::from("a")),
value: Some(Bytes::from("1")),
headers: vec![],
},
Record {
attributes: 0,
timestamp_delta: 0,
offset_delta: 1,
key: Some(Bytes::from("b")),
value: Some(Bytes::from("2")),
headers: vec![],
},
Record {
attributes: 0,
timestamp_delta: 0,
offset_delta: 2,
key: Some(Bytes::from("a")),
value: Some(Bytes::from("3")),
headers: vec![],
},
];
let raw = RecordBatch::serialize(0, records, Compression::None, 100, 0, 0, 0, 0);
let compacted = LogCompactor::compact_batches(raw).unwrap();
let vals: Vec<Option<Bytes>> = compacted
.iter()
.map(|r| r.value.clone())
.collect();
assert!(vals.contains(&Some(Bytes::from("3"))));
assert!(vals.contains(&Some(Bytes::from("2"))));
assert!(!vals.contains(&Some(Bytes::from("1"))));
}
}