use crate::ObjectLogError;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Mutex;
#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
pub struct PartitionKey(pub String);
impl PartitionKey {
pub fn as_str(&self) -> &str {
&self.0
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BatchLocation {
pub object_id: String,
pub byte_start: u32,
pub byte_len: u32,
}
pub struct CommitBatch<'a, M> {
pub partition: PartitionKey,
pub record_count: i32,
pub location: BatchLocation,
pub meta: &'a M,
pub epoch: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum CommitOutcome {
Assigned {
base_offset: i64,
record_count: i32,
},
Duplicate {
base_offset: i64,
},
Rejected {
reason: String,
},
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct IndexEntry {
pub location: BatchLocation,
pub base_offset: i64,
pub record_count: i32,
}
pub trait Sequencer: Send + Sync {
type Meta: Send + Sync;
fn supports_multi_object_commit(&self) -> bool {
false
}
fn commit(
&self,
batches: &[CommitBatch<'_, Self::Meta>],
) -> Result<Vec<CommitOutcome>, ObjectLogError>;
fn lookup(
&self,
partition: &PartitionKey,
fetch_offset: i64,
) -> Result<Vec<IndexEntry>, ObjectLogError>;
fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError>;
fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError>;
fn truncate_before(
&self,
partition: &PartitionKey,
offset: i64,
) -> Result<Vec<String>, ObjectLogError>;
fn fence_epoch(
&self,
partition: &PartitionKey,
expected: u64,
new_epoch: u64,
) -> Result<(), ObjectLogError> {
let _ = (partition, expected, new_epoch);
Ok(())
}
fn refresh_partition(&self, partition: &PartitionKey) -> Result<(), ObjectLogError> {
let _ = partition;
Ok(())
}
}
#[derive(Default)]
struct PartState {
next_offset: i64,
log_start: i64,
entries: Vec<IndexEntry>,
}
#[derive(Default)]
pub struct InMemorySequencer {
state: Mutex<HashMap<PartitionKey, PartState>>,
}
impl InMemorySequencer {
pub fn new() -> Self {
Self::default()
}
pub fn live_object_ids(&self) -> HashSet<String> {
let state = self.state.lock().expect("poisoned");
let mut live = HashSet::new();
for ps in state.values() {
for e in &ps.entries {
live.insert(e.location.object_id.clone());
}
}
live
}
}
impl Sequencer for InMemorySequencer {
type Meta = ();
fn supports_multi_object_commit(&self) -> bool {
true
}
fn commit(
&self,
batches: &[CommitBatch<'_, Self::Meta>],
) -> Result<Vec<CommitOutcome>, ObjectLogError> {
let mut state = self.state.lock().expect("poisoned");
let mut outcomes = Vec::with_capacity(batches.len());
for cb in batches {
let ps = state.entry(cb.partition.clone()).or_default();
let base = ps.next_offset;
ps.entries.push(IndexEntry {
location: cb.location.clone(),
base_offset: base,
record_count: cb.record_count,
});
ps.next_offset = base + cb.record_count as i64;
outcomes.push(CommitOutcome::Assigned {
base_offset: base,
record_count: cb.record_count,
});
}
Ok(outcomes)
}
fn lookup(
&self,
partition: &PartitionKey,
fetch_offset: i64,
) -> Result<Vec<IndexEntry>, ObjectLogError> {
let state = self.state.lock().expect("poisoned");
let Some(ps) = state.get(partition) else {
return Ok(Vec::new());
};
Ok(ps
.entries
.iter()
.filter(|e| e.base_offset + e.record_count as i64 > fetch_offset)
.cloned()
.collect())
}
fn high_watermark(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
Ok(self
.state
.lock()
.expect("poisoned")
.get(partition)
.map_or(0, |ps| ps.next_offset))
}
fn log_start_offset(&self, partition: &PartitionKey) -> Result<i64, ObjectLogError> {
Ok(self
.state
.lock()
.expect("poisoned")
.get(partition)
.map_or(0, |ps| ps.log_start))
}
fn truncate_before(
&self,
partition: &PartitionKey,
offset: i64,
) -> Result<Vec<String>, ObjectLogError> {
let mut state = self.state.lock().expect("poisoned");
let mut dropped: Vec<String> = Vec::new();
match state.get_mut(partition) {
Some(ps) => {
for e in ps.entries.iter() {
if e.base_offset + e.record_count as i64 <= offset {
dropped.push(e.location.object_id.clone());
}
}
ps.entries
.retain(|e| e.base_offset + e.record_count as i64 > offset);
if offset > ps.log_start {
ps.log_start = offset.min(ps.next_offset);
}
}
None => return Ok(Vec::new()),
}
let mut live: HashSet<String> = HashSet::new();
for ps in state.values() {
for e in &ps.entries {
live.insert(e.location.object_id.clone());
}
}
let mut dead: Vec<String> = Vec::new();
let mut seen: HashSet<String> = HashSet::new();
for oid in dropped {
if !live.contains(&oid) && seen.insert(oid.clone()) {
dead.push(oid);
}
}
Ok(dead)
}
}