use super::segment::LogSegment;
use crate::config::BrokerConfig;
use crate::protocol::{RecordBatch, RecordBatchReader};
use anyhow::Result;
use bytes::Bytes;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
#[derive(Debug, Clone)]
pub struct AppendResult {
pub base_offset: i64,
pub last_offset: i64,
pub log_append_time_ms: i64,
pub log_start_offset: i64,
}
pub struct PartitionLog {
pub topic: String,
pub partition: i32,
pub dir: PathBuf,
segments: BTreeMap<i64, LogSegment>,
pub high_watermark: i64,
pub log_start_offset: i64,
config: Arc<BrokerConfig>,
}
impl PartitionLog {
pub fn open(topic: &str, partition: i32, dir: &PathBuf, config: Arc<BrokerConfig>) -> Result<Self> {
std::fs::create_dir_all(dir)?;
let mut segments = BTreeMap::new();
let mut base_offsets: Vec<i64> = Vec::new();
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().to_string();
if name.ends_with(".log") {
let base = name.trim_end_matches(".log").parse::<i64>().unwrap_or(0);
base_offsets.push(base);
}
}
base_offsets.sort_unstable();
let mut log_start_offset = i64::MAX;
for base in &base_offsets {
let seg = LogSegment::open(
dir,
*base,
config.index_interval_bytes,
config.log_segment_bytes,
config.log_roll_ms,
)?;
if *base < log_start_offset {
log_start_offset = *base;
}
segments.insert(*base, seg);
}
if segments.is_empty() {
let seg = LogSegment::open(
dir,
0,
config.index_interval_bytes,
config.log_segment_bytes,
config.log_roll_ms,
)?;
log_start_offset = 0;
segments.insert(0, seg);
}
let leo = segments
.iter()
.next_back()
.map(|(_, s)| s.max_offset + 1)
.unwrap_or(0);
let log = Self {
topic: topic.to_string(),
partition,
dir: dir.clone(),
segments,
high_watermark: leo,
log_start_offset: if log_start_offset == i64::MAX { 0 } else { log_start_offset },
config,
};
log.recover()?;
Ok(log)
}
fn recover(&self) -> Result<()> {
Ok(())
}
pub fn append(&mut self, batches: &[Bytes]) -> Result<Vec<AppendResult>> {
let mut results = Vec::new();
for raw in batches {
let mut reader = RecordBatchReader::new(raw.clone());
while let Some((batch, _)) = reader.next_batch()? {
let offset = self.next_offset();
{
let active = self.active_segment();
if !active.closed && active.should_roll() {
active.close()?;
}
}
let rebased = rebase_batch(&batch, offset);
let active = self.active_segment();
let r = active.append(&rebased)?;
self.high_watermark = active.max_offset + 1;
results.push(r);
}
}
Ok(results)
}
pub fn read(&self, start_offset: i64, max_bytes: usize) -> Bytes {
if start_offset < self.log_start_offset {
return Bytes::new();
}
if let Some(seg) = self.find_segment(start_offset) {
seg.read(start_offset, max_bytes).unwrap_or_default()
} else {
Bytes::new()
}
}
fn active_segment(&mut self) -> &mut LogSegment {
let last_key = self.segments.iter().next_back().map(|(k, _)| *k).unwrap_or(0);
if let Some(seg) = self.segments.get_mut(&last_key) {
seg
} else {
unreachable!()
}
}
fn find_segment(&self, offset: i64) -> Option<&LogSegment> {
self.segments
.range(..=offset)
.next_back()
.map(|(_, s)| s)
}
pub fn next_offset(&self) -> i64 {
self.high_watermark
}
pub fn leo(&self) -> i64 {
self.high_watermark
}
pub fn log_start(&self) -> i64 {
self.log_start_offset
}
pub fn high_watermark(&self) -> i64 {
self.high_watermark
}
pub fn flush(&mut self) -> Result<()> {
for seg in self.segments.values_mut() {
seg.flush_log()?;
}
Ok(())
}
pub fn cleanup_retention(&mut self) -> Result<()> {
if self.config.cleanup_policy != "delete" {
return Ok(());
}
let retention_hours = self.config.log_retention_hours;
let retention_bytes = self.config.log_retention_bytes;
let total: u64 = self.segments.values().map(|s| s.size_in_bytes).sum();
let keys: Vec<i64> = self.segments.keys().copied().collect();
for base in &keys {
if *base == *self.segments.iter().next_back().map(|(k, _)| k).unwrap_or(&0) {
break; }
let seg = self.segments.get(base).unwrap();
let expired_by_time = seg.is_expired(retention_hours);
let expired_by_size = retention_bytes >= 0 && total > retention_bytes as u64;
if expired_by_time || expired_by_size {
seg.delete_files()?;
self.segments.remove(base);
if let Some(first) = self.segments.iter().next().map(|(k, _)| *k) {
self.log_start_offset = first;
}
}
}
Ok(())
}
pub fn segment_count(&self) -> usize {
self.segments.len()
}
}
fn rebase_batch(batch: &RecordBatch, new_base_offset: i64) -> RecordBatch {
let mut records = batch.records.clone();
for (i, r) in records.iter_mut().enumerate() {
r.offset_delta = i as i32;
}
let raw = crate::protocol::recordbatch::RecordBatch::serialize(
new_base_offset,
records,
batch.compression(),
batch.base_timestamp,
batch.producer_id,
batch.producer_epoch,
batch.base_sequence,
batch.partition_leader_epoch,
);
RecordBatch::parse(raw).expect("重写后批次必然合法")
}