use crate::config::BrokerConfig;
use crate::storage::PartitionLog;
use anyhow::Result;
use bytes::Bytes;
use dashmap::DashMap;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::RwLock;
#[derive(Debug, Clone)]
pub struct PartitionInfo {
pub topic: String,
pub partition: i32,
pub leader: i32,
pub replicas: Vec<i32>,
pub isr: Vec<i32>,
}
#[derive(Debug, Clone)]
pub struct TopicInfo {
pub name: String,
pub is_internal: bool,
pub partitions: Vec<PartitionInfo>,
}
pub struct MetadataManager {
config: Arc<BrokerConfig>,
logs: DashMap<String, DashMap<i32, Arc<RwLock<PartitionLog>>>>,
topics: DashMap<String, TopicInfo>,
}
impl MetadataManager {
pub fn new(config: Arc<BrokerConfig>) -> Self {
Self {
config,
logs: DashMap::new(),
topics: DashMap::new(),
}
}
pub fn init(&self) -> Result<()> {
for dir in &self.config.log_dirs {
let root = PathBuf::from(dir);
if !root.exists() {
std::fs::create_dir_all(&root)?;
continue;
}
for entry in std::fs::read_dir(&root)? {
let entry = entry?;
if !entry.file_type()?.is_dir() {
continue;
}
let name = entry.file_name().to_string_lossy().to_string();
if let Some((topic, partition)) = parse_topic_partition(&name) {
self.open_partition(&topic, partition)?;
}
}
}
Ok(())
}
pub fn open_partition(&self, topic: &str, partition: i32) -> Result<()> {
let dir = self.partition_dir(topic, partition);
let log = PartitionLog::open(topic, partition, &dir, self.config.clone())?;
self.logs
.entry(topic.to_string())
.or_insert_with(DashMap::new)
.insert(partition, Arc::new(RwLock::new(log)));
let info = PartitionInfo {
topic: topic.to_string(),
partition,
leader: self.config.broker_id,
replicas: vec![self.config.broker_id],
isr: vec![self.config.broker_id],
};
self.topics
.entry(topic.to_string())
.or_insert_with(|| TopicInfo {
name: topic.to_string(),
is_internal: false,
partitions: vec![],
})
.partitions
.push(info);
Ok(())
}
pub fn partition_dir(&self, topic: &str, partition: i32) -> PathBuf {
self.config
.primary_log_dir()
.join(format!("{topic}-{partition}"))
}
pub fn create_topic(&self, topic: &str, num_partitions: i32, is_internal: bool) -> Result<()> {
self.topics.entry(topic.to_string()).or_insert_with(|| TopicInfo {
name: topic.to_string(),
is_internal,
partitions: vec![],
});
for p in 0..num_partitions {
if self.partition_exists(topic, p) {
continue;
}
self.open_partition(topic, p)?;
}
Ok(())
}
pub fn delete_topic(&self, topic: &str) -> Result<()> {
if let Some(logs) = self.logs.remove(topic) {
for entry in logs.1.iter() {
let dir = self.partition_dir(topic, *entry.key());
let _ = std::fs::remove_dir_all(&dir);
}
}
self.topics.remove(topic);
Ok(())
}
pub async fn append_records(
&self,
topic: &str,
partition: i32,
batches: &[Bytes],
) -> Result<Vec<(i64, i64)>> {
let log = self.get_log_arc(topic, partition)
.ok_or_else(|| anyhow::anyhow!("分区 {topic}-{partition} 不存在"))?;
let mut guard = log.write().await;
let results = guard.append(batches)?;
Ok(results.iter().map(|r| (r.base_offset, r.last_offset)).collect())
}
pub fn append_records_sync(
&self,
topic: &str,
partition: i32,
batches: &[Bytes],
) -> Result<Vec<(i64, i64)>> {
let log = self
.get_log_arc(topic, partition)
.ok_or_else(|| anyhow::anyhow!("分区 {topic}-{partition} 不存在"))?;
let mut guard = log
.try_write()
.map_err(|_| anyhow::anyhow!("分区 {topic}-{partition} 写锁被占用"))?;
let results = guard.append(batches)?;
Ok(results.iter().map(|r| (r.base_offset, r.last_offset)).collect())
}
pub async fn read_records(
&self,
topic: &str,
partition: i32,
start_offset: i64,
max_bytes: usize,
) -> Option<(Bytes, i64)> {
let log = self.get_log_arc(topic, partition)?;
let guard = log.read().await;
let hw = guard.high_watermark();
let data = guard.read(start_offset, max_bytes);
Some((data, hw))
}
pub fn read_records_sync(
&self,
topic: &str,
partition: i32,
start_offset: i64,
max_bytes: usize,
) -> Option<(Bytes, i64)> {
let log = self.get_log_arc(topic, partition)?;
let guard = log.try_read().ok()?;
let hw = guard.high_watermark();
let data = guard.read(start_offset, max_bytes);
Some((data, hw))
}
pub async fn partition_leo(&self, topic: &str, partition: i32) -> Option<i64> {
let log = self.get_log_arc(topic, partition)?;
Some(log.read().await.leo())
}
pub fn partition_leo_sync(&self, topic: &str, partition: i32) -> Option<i64> {
let log = self.get_log_arc(topic, partition)?;
log.try_read().ok().map(|l| l.leo())
}
pub fn partition_log_start_sync(&self, topic: &str, partition: i32) -> Option<i64> {
let log = self.get_log_arc(topic, partition)?;
log.try_read().ok().map(|l| l.log_start())
}
pub fn get_log_arc(&self, topic: &str, partition: i32) -> Option<Arc<RwLock<PartitionLog>>> {
self.logs.get(topic)?.get(&partition).map(|l| l.clone())
}
pub fn topic_list(&self) -> Vec<TopicInfo> {
self.topics.iter().map(|t| t.value().clone()).collect()
}
pub fn get_topic(&self, topic: &str) -> Option<TopicInfo> {
self.topics.get(topic).map(|t| t.value().clone())
}
pub fn partition_exists(&self, topic: &str, partition: i32) -> bool {
self.logs
.get(topic)
.map(|l| l.contains_key(&partition))
.unwrap_or(false)
}
pub fn topic_count(&self) -> usize {
self.topics.len()
}
pub fn default_partitions(&self) -> i32 {
self.config.num_partitions.max(1)
}
pub fn partitions_of(&self, topic: &str) -> i32 {
self.topics
.get(topic)
.map(|t| t.value().partitions.len() as i32)
.unwrap_or(0)
}
pub fn create_internal_topic(&self, name: &str, partitions: i32) -> Result<()> {
self.create_topic(name, partitions, true)
}
}
fn parse_topic_partition(name: &str) -> Option<(String, i32)> {
let idx = name.rfind('-')?;
let (topic, part) = name.split_at(idx);
let part = part.trim_start_matches('-').parse::<i32>().ok()?;
Some((topic.to_string(), part))
}