Skip to main content

object_log/
engine.rs

1//! The buffered, multiplexing log engine.
2
3use crate::budget::{
4    BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8    BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21/// The durability point a [`LogEngine::produce`] call resolves at.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24    /// Resolve as soon as the batch is buffered (fire-and-forget; may be lost on
25    /// crash before the flush). No offset is returned.
26    Buffered,
27    /// Resolve once the containing object is durably PUT (survives crash). No
28    /// offset yet — the commit has not run.
29    Durable,
30    /// Resolve once the batch is durably PUT **and** sequenced (has a stable
31    /// offset). This is the strong, no-loss level.
32    Sequenced,
33}
34
35/// Flush-trigger policy for the engine's group-commit buffer.
36#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38    /// Hard ceiling on one sealed object's payload size (memory / object-store
39    /// physics). **Not** the primary packing control: under normal load packing
40    /// is `ingest_rate × linger`. Keep this high enough that **linger binds**
41    /// before size (default **1 GiB**). Lower only for tight RAM or backend caps.
42    pub max_bytes: usize,
43    /// Flush once this many batches are buffered (secondary ceiling).
44    pub max_batches: usize,
45    /// **Maximum** time a batch may wait before a deadline flush (hard ceiling).
46    /// This is the latency↔throughput control surface: longer wait ⇒ more bytes
47    /// per seal ⇒ fewer durable ops/s. The budget controller may use a shorter
48    /// *effective* linger when media is idle (TD-004). `ZERO` = seal as soon as
49    /// any data is buffered (no co-buffer wait). Default `50ms`.
50    pub linger: Duration,
51    /// Max sealed objects PUT concurrently. Default **1** (single-flight bulk
52    /// path). Raise for parallel S3 PUTs. Also bounds ready objects per atomic
53    /// commit when the sequencer opts in to multi-object commits.
54    pub max_inflight_flushes: usize,
55    /// Max bytes in the mutable queue plus in-flight seals. Producers block when
56    /// exceeded. Default **2 GiB**.
57    pub max_buffered_bytes: usize,
58    /// Durable-ops budget controller (default on for Fjord).
59    pub budget: BudgetConfig,
60}
61
62impl Default for FlushConfig {
63    fn default() -> Self {
64        Self {
65            // Physics/safety ceiling so linger defines segment size under load.
66            max_bytes: 1024 * 1024 * 1024,
67            max_batches: 10_000,
68            linger: Duration::from_millis(50),
69            // 1 = single-flight seal (best Local bulk); raise for parallel S3 PUTs.
70            max_inflight_flushes: 1,
71            max_buffered_bytes: 2 * 1024 * 1024 * 1024,
72            budget: BudgetConfig::default(),
73        }
74    }
75}
76
77/// Outcome of a [`LogEngine::produce`] call.
78#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct AppendOutcome {
80    /// First assigned offset, when resolved at [`Durability::Sequenced`].
81    pub base_offset: Option<i64>,
82    /// Last assigned offset, when resolved at [`Durability::Sequenced`].
83    pub last_offset: Option<i64>,
84    /// Whether the batch is durably stored.
85    pub durable: bool,
86    /// Whether the batch has been sequenced (has an offset).
87    pub sequenced: bool,
88}
89
90/// A batch read back by [`LogEngine::fetch`], with its assigned base offset.
91#[derive(Clone, Debug, PartialEq, Eq)]
92pub struct FetchedBatch {
93    /// First offset of the batch.
94    pub base_offset: i64,
95    /// Number of records in the batch.
96    pub record_count: i32,
97    /// The opaque batch payload.
98    pub payload: Bytes,
99}
100
101/// Snapshot of the engine's buffering envelope.
102#[derive(Clone, Debug, PartialEq, Eq)]
103pub struct BufferStats {
104    /// Bytes still waiting in the mutable accumulation queue.
105    pub queued_bytes: usize,
106    /// Bytes owned by queued plus sealed in-flight flush work.
107    pub bytes_in_use: usize,
108    /// Batches still waiting in the mutable accumulation queue.
109    pub queued_batches: usize,
110    /// Configured upper bound for `bytes_in_use`.
111    pub max_buffered_bytes: usize,
112}
113
114type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
115
116fn retryable_storage_error(err: &ObjectLogError) -> bool {
117    matches!(err, ObjectLogError::StorageUnavailable(_))
118}
119
120async fn retry_delay(attempt: usize) {
121    let multiplier = 1u32 << attempt.min(5);
122    tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
123}
124
125async fn put_chunks_with_retries(
126    blob: &Arc<dyn BlobStore>,
127    key: &str,
128    chunks: Vec<Bytes>,
129) -> Result<(), ObjectLogError> {
130    // `Bytes::clone` is refcount-only; Local put_chunks streams without re-merge.
131    let mut attempt = 0usize;
132    loop {
133        match blob.put_chunks(key, chunks.clone()).await {
134            Ok(()) => return Ok(()),
135            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
136                retry_delay(attempt).await;
137                attempt += 1;
138            }
139            Err(err) => return Err(err),
140        }
141    }
142}
143
144async fn get_range_with_retries(
145    blob: &Arc<dyn BlobStore>,
146    key: &str,
147    range: std::ops::Range<u64>,
148) -> Result<Option<Bytes>, ObjectLogError> {
149    let mut attempt = 0usize;
150    loop {
151        match blob.get_range(key, range.clone()).await {
152            Ok(bytes) => return Ok(bytes),
153            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
154                retry_delay(attempt).await;
155                attempt += 1;
156            }
157            Err(err) => return Err(err),
158        }
159    }
160}
161
162struct Pending<M> {
163    partition: PartitionKey,
164    record_count: i32,
165    payload: Bytes,
166    meta: M,
167    /// Fence epoch carried into [`CommitBatch::epoch`].
168    epoch: u64,
169    durability: Durability,
170    responder: Option<Responder>,
171    /// Monotonic enqueue id for [`LogEngine::flush`] barriers.
172    seq: u64,
173}
174
175struct FlushWork<M> {
176    batch: Vec<Pending<M>>,
177    locations: Vec<BatchLocation>,
178    responders: Vec<(Durability, Option<Responder>)>,
179    bytes: usize,
180    /// Highest enqueue seq included in this flush object.
181    max_seq: u64,
182    /// Durable data objects represented by this ordered commit group.
183    data_objects: usize,
184    put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
185    put_started: Instant,
186    put_result: Option<Result<Duration, ObjectLogError>>,
187}
188
189struct CommitJob {
190    task: TokioJoinHandle<usize>,
191    first_seq: u64,
192}
193
194enum TakeBatch<M> {
195    Batch(Vec<Pending<M>>),
196    Empty,
197    Shutdown,
198}
199
200type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
201
202struct Queue<M> {
203    items: VecDeque<Pending<M>>,
204    bytes: usize,
205    bytes_in_use: usize,
206    shutdown: bool,
207    /// Next seq to assign on enqueue.
208    next_seq: u64,
209    /// Highest settled enqueue sequence; failures are retained separately.
210    completed_through: u64,
211    /// Earliest failed enqueue prefix. A later success cannot make it durable.
212    first_failed: Option<(u64, ObjectLogError)>,
213    /// When true, take_batch flushes even if under size/linger.
214    force_flush: bool,
215    /// Waiters: (barrier_seq inclusive, responder).
216    flush_waiters: Vec<FlushWaiter>,
217    /// Last successful produce enqueue (for idle early-flush).
218    last_enqueue: Option<Instant>,
219    /// Enqueue time of the front item (linger deadline = oldest + linger).
220    oldest_enqueue: Option<Instant>,
221}
222
223struct Shared<M> {
224    queue: Mutex<Queue<M>>,
225    cv: Condvar,
226    max_buffered_bytes: usize,
227    /// Budget controller + inspectable counters (TD-004).
228    budget: Mutex<BudgetRuntime>,
229    /// Max linger (operator ceiling) and budget config copy for the flush loop.
230    flush_config: FlushConfig,
231}
232
233/// A buffered, multiplexing append-log engine over a [`BlobStore`], with
234/// sequencing delegated to a [`Sequencer`].
235///
236/// `produce` group-commits: many batches across many partitions are multiplexed
237/// into one object, PUT durably, then handed to the sequencer in a single call —
238/// so PUT count is decoupled from produce count. A single flush worker preserves
239/// per-[`PartitionKey`] arrival order and never splits a partition across
240/// concurrent commits.
241pub struct LogEngine<S: Sequencer> {
242    shared: Arc<Shared<S::Meta>>,
243    blob: Arc<dyn BlobStore>,
244    sequencer: Arc<S>,
245    /// Data-object key prefix (`<prefix><counter:020>`, or
246    /// `<prefix><writer>/<epoch>/<counter>` when a writer id is set).
247    key_prefix: String,
248    writer_id: String,
249    flush_thread: Option<JoinHandle<()>>,
250}
251
252impl<S> LogEngine<S>
253where
254    S: Sequencer + 'static,
255    S::Meta: Send + 'static,
256{
257    /// Create an engine over `blob` and `sequencer` with the given flush policy.
258    /// Objects are keyed `<key_prefix><counter>`; pick a prefix unique to this
259    /// engine instance if several share a store.
260    pub fn new(
261        blob: Arc<dyn BlobStore>,
262        sequencer: Arc<S>,
263        config: FlushConfig,
264        key_prefix: impl Into<String>,
265    ) -> Self {
266        Self::new_with_writer(blob, sequencer, config, key_prefix, "")
267    }
268
269    /// Like [`new`](Self::new), but data objects are
270    /// `<key_prefix><writer_id>/<epoch>/<counter>` so two writers cannot
271    /// overwrite the same key. `writer_id` must not contain `/` or `..`.
272    pub fn new_with_writer(
273        blob: Arc<dyn BlobStore>,
274        sequencer: Arc<S>,
275        config: FlushConfig,
276        key_prefix: impl Into<String>,
277        writer_id: impl Into<String>,
278    ) -> Self {
279        if let Err(msg) = config.budget.validate() {
280            panic!("invalid FlushConfig.budget: {msg}");
281        }
282        if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
283            eprintln!("object-log flush config: {config:?}");
284        }
285        let budget_rt = BudgetRuntime::new(config.budget);
286        let shared = Arc::new(Shared {
287            queue: Mutex::new(Queue {
288                items: VecDeque::new(),
289                bytes: 0,
290                bytes_in_use: 0,
291                shutdown: false,
292                next_seq: 1, // first enqueued batch gets seq 1
293                completed_through: 0,
294                first_failed: None,
295                force_flush: false,
296                flush_waiters: Vec::new(),
297                last_enqueue: None,
298                oldest_enqueue: None,
299            }),
300            cv: Condvar::new(),
301            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
302            budget: Mutex::new(budget_rt),
303            flush_config: config,
304        });
305        let key_prefix = key_prefix.into();
306        let writer_id = writer_id.into();
307        let flush_thread = {
308            let shared = Arc::clone(&shared);
309            let blob = Arc::clone(&blob);
310            let sequencer = Arc::clone(&sequencer);
311            let prefix = key_prefix.clone();
312            let writer_id = writer_id.clone();
313            std::thread::Builder::new()
314                .name("object-log-flush".into())
315                .spawn(move || flush_loop(shared, blob, sequencer, config, prefix, writer_id))
316                .expect("spawn flush thread")
317        };
318        Self {
319            writer_id,
320            shared,
321            blob,
322            sequencer,
323            key_prefix,
324            flush_thread: Some(flush_thread),
325        }
326    }
327
328    /// Prefix used for data objects written by this engine.
329    pub fn data_prefix(&self) -> &str {
330        &self.key_prefix
331    }
332
333    /// Borrow the engine's blob store (e.g. for orphan reaping or inspection).
334    pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
335        &self.blob
336    }
337
338    /// Borrow the engine's sequencer.
339    pub fn sequencer(&self) -> &Arc<S> {
340        &self.sequencer
341    }
342
343    /// Writer id used in data-object keys. Empty when constructed with [`new`](Self::new).
344    pub fn writer_id(&self) -> &str {
345        &self.writer_id
346    }
347
348    /// Buffer a batch and resolve at the requested [`Durability`].
349    ///
350    /// The batch's fence epoch is `0`. Use [`produce_at_epoch`](Self::produce_at_epoch)
351    /// when the partition index is fenced.
352    pub async fn produce(
353        &self,
354        partition: PartitionKey,
355        payload: Bytes,
356        record_count: i32,
357        meta: S::Meta,
358        durability: Durability,
359    ) -> Result<AppendOutcome, ObjectLogError> {
360        self.produce_at_epoch(partition, payload, record_count, meta, durability, 0)
361            .await
362    }
363
364    /// [`produce`](Self::produce) at a caller-supplied fence epoch.
365    pub async fn produce_at_epoch(
366        &self,
367        partition: PartitionKey,
368        payload: Bytes,
369        record_count: i32,
370        meta: S::Meta,
371        durability: Durability,
372        epoch: u64,
373    ) -> Result<AppendOutcome, ObjectLogError> {
374        if record_count <= 0 {
375            return Err(ObjectLogError::InvalidBatch(
376                "record_count must be > 0".into(),
377            ));
378        }
379        if matches!(durability, Durability::Buffered) {
380            self.enqueue(Pending {
381                partition,
382                record_count,
383                payload,
384                meta,
385                epoch,
386                durability,
387                responder: None,
388                seq: 0, // filled in enqueue
389            })?;
390            return Ok(AppendOutcome {
391                base_offset: None,
392                last_offset: None,
393                durable: false,
394                sequenced: false,
395            });
396        }
397        // fail_closed: reserve predicted media ops before waiting on flush.
398        {
399            let mut budget = self.shared.budget.lock().expect("poisoned");
400            if budget.config.enabled
401                && budget.config.mode == BudgetMode::FailClosed
402                && !budget.reserve_for_fail_closed(Instant::now())
403            {
404                return Err(ObjectLogError::BudgetExceeded(
405                    "insufficient durable-ops budget to admit produce".into(),
406                ));
407            }
408        }
409        // budget_priority: wait briefly for tokens when empty.
410        if self.shared.flush_config.budget.enabled
411            && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
412        {
413            let timeout = self.shared.flush_config.budget.admission_timeout;
414            let deadline = Instant::now() + timeout;
415            loop {
416                let mut budget = self.shared.budget.lock().expect("poisoned");
417                if budget.can_admit_now(Instant::now()) {
418                    break;
419                }
420                drop(budget);
421                if Instant::now() >= deadline {
422                    return Err(ObjectLogError::BudgetExceeded(
423                        "timed out waiting for durable-ops budget".into(),
424                    ));
425                }
426                std::thread::sleep(Duration::from_millis(1));
427            }
428        }
429        let (tx, rx) = oneshot::channel();
430        self.enqueue(Pending {
431            partition,
432            record_count,
433            payload,
434            meta,
435            epoch,
436            durability,
437            responder: Some(tx),
438            seq: 0,
439        })?;
440        rx.await
441            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
442    }
443
444    /// Seal every batch enqueued **at or before** this call (barrier).
445    ///
446    /// Use after [`Durability::Buffered`] produces to wait for durable PUT (and
447    /// sequencing) of that prior work. Concurrent produces enqueued after this
448    /// call are not required to finish. An empty successful buffer returns immediately.
449    /// A failed PUT or commit remains an error for every barrier covering that
450    /// enqueue prefix, even after later appends succeed. Reopen the engine to
451    /// establish a new prefix from its successfully committed index.
452    pub async fn flush(&self) -> Result<(), ObjectLogError> {
453        let rx = {
454            let mut q = self.shared.queue.lock().expect("poisoned");
455            if q.shutdown {
456                return Err(ObjectLogError::Sequencer("engine shutting down".into()));
457            }
458            if q.next_seq == 1 {
459                // Nothing has ever been enqueued.
460                return Ok(());
461            }
462            let barrier = q.next_seq - 1;
463            if let Some((sequence, error)) = &q.first_failed {
464                if *sequence <= barrier {
465                    return Err(error.clone());
466                }
467            }
468            if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
469                return Ok(());
470            }
471            let (tx, rx) = oneshot::channel();
472            q.force_flush = true;
473            q.flush_waiters.push((barrier, tx));
474            self.shared.cv.notify_all();
475            rx
476        };
477        rx.await
478            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
479    }
480
481    fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
482        let item_len = item.payload.len();
483        let max_buffered_bytes = self.shared.max_buffered_bytes;
484        let shared = Arc::clone(&self.shared);
485        let mut q = shared.queue.lock().expect("poisoned");
486        while !q.shutdown
487            && q.bytes_in_use > 0
488            && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
489        {
490            q = shared.cv.wait(q).expect("poisoned");
491        }
492        if q.shutdown {
493            return Err(ObjectLogError::Sequencer("engine shutting down".into()));
494        }
495        let now = Instant::now();
496        item.seq = q.next_seq;
497        q.next_seq = q.next_seq.saturating_add(1);
498        q.bytes += item_len;
499        q.bytes_in_use += item_len;
500        q.last_enqueue = Some(now);
501        if q.oldest_enqueue.is_none() {
502            q.oldest_enqueue = Some(now);
503        }
504        q.items.push_back(item);
505        shared.cv.notify_all();
506        Ok(())
507    }
508
509    /// Read batches covering offsets at/after `offset`, up to ~`max_bytes`.
510    pub async fn fetch(
511        &self,
512        partition: &PartitionKey,
513        offset: i64,
514        max_bytes: usize,
515    ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
516        let entries = self.sequencer.lookup(partition, offset)?;
517        let mut out = Vec::new();
518        let mut total = 0usize;
519        for e in entries {
520            if total >= max_bytes && !out.is_empty() {
521                break;
522            }
523            let batch = self.load_entry(&e).await?;
524            total += batch.payload.len();
525            out.push(batch);
526        }
527        Ok(out)
528    }
529
530    /// Stream batches from `offset` onward without materializing a full `Vec`.
531    ///
532    /// Calls `visit` once per index entry in order. Suitable for wide offset
533    /// windows (bounded-RAM replay). Stops and returns the error if `visit`
534    /// fails. Unlike [`Self::fetch`], there is no byte budget — visit every remaining
535    /// batch (or stop yourself inside `visit`).
536    pub async fn fetch_stream<F>(
537        &self,
538        partition: &PartitionKey,
539        offset: i64,
540        mut visit: F,
541    ) -> Result<(), ObjectLogError>
542    where
543        F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
544    {
545        let entries = self.sequencer.lookup(partition, offset)?;
546        for e in entries {
547            visit(self.load_entry(&e).await?)?;
548        }
549        Ok(())
550    }
551
552    async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
553        let start = e.location.byte_start as u64;
554        let end = start + e.location.byte_len as u64;
555        let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
556            .await?
557            .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
558        Ok(FetchedBatch {
559            base_offset: e.base_offset,
560            record_count: e.record_count,
561            payload: bytes,
562        })
563    }
564
565    /// Drop the partition's log below `offset` and delete any object that thereby
566    /// becomes fully unreferenced.
567    pub async fn truncate_before(
568        &self,
569        partition: &PartitionKey,
570        offset: i64,
571    ) -> Result<(), ObjectLogError> {
572        let dead = self.sequencer.truncate_before(partition, offset)?;
573        for object_id in dead {
574            self.blob.delete(&object_id).await?;
575        }
576        Ok(())
577    }
578
579    /// Delete data-prefix objects that are not in `live`.
580    ///
581    /// **Safety:** call only when this engine (and any other writer using the
582    /// same prefix) is quiescent — otherwise an in-flight put that has not yet
583    /// been committed may be deleted as an "orphan". Prefer running offline or
584    /// after drop. Does not touch keys outside [`data_prefix`](Self::data_prefix)
585    /// (e.g. a separate manifest prefix is safe).
586    ///
587    /// Returns the deleted object keys.
588    pub async fn reap_orphans(
589        &self,
590        live: &HashSet<String>,
591    ) -> Result<Vec<String>, ObjectLogError> {
592        reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
593    }
594
595    /// Return a point-in-time snapshot of queued and in-flight payload bytes.
596    pub fn buffer_stats(&self) -> BufferStats {
597        let q = self.shared.queue.lock().expect("poisoned");
598        BufferStats {
599            queued_bytes: q.bytes,
600            bytes_in_use: q.bytes_in_use,
601            queued_batches: q.items.len(),
602            max_buffered_bytes: self.shared.max_buffered_bytes,
603        }
604    }
605
606    /// Inspect budget controller layers and counters (TD-004).
607    pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
608        let now = Instant::now();
609        let mut budget = self.shared.budget.lock().expect("poisoned");
610        budget.refill(now);
611        let effective = budget.effective_budget_per_sec;
612        let reason = if !budget.config.enabled {
613            EffectiveReason::Disabled
614        } else if budget.config.budget_per_sec_cap.is_some()
615            && budget
616                .config
617                .budget_per_sec_cap
618                .is_some_and(|c| (effective - c).abs() < 1e-9)
619        {
620            EffectiveReason::ConfigCap
621        } else if budget.ongoing_capacity.is_some() {
622            EffectiveReason::Ongoing
623        } else if budget.startup_capacity.is_some() {
624            EffectiveReason::StartupProbe
625        } else {
626            EffectiveReason::DefaultCapacity
627        };
628        let (queued, last_enq) = {
629            let q = self.shared.queue.lock().expect("poisoned");
630            (q.bytes, q.last_enqueue)
631        };
632        let early = budget.allow_early_flush(now, queued, last_enq);
633        let max_linger = self.shared.flush_config.linger;
634        let eff_linger = if !budget.config.enabled {
635            max_linger
636        } else if early || max_linger.is_zero() {
637            Duration::ZERO
638        } else {
639            max_linger
640        };
641        PipelineSnapshot {
642            budget_per_sec: EffectiveKnob {
643                configured: budget.config.budget_per_sec_cap,
644                startup_measured: budget.startup_capacity,
645                ongoing_measured: budget.ongoing_capacity,
646                effective,
647                reason,
648            },
649            effective_linger_ms: EffectiveKnob {
650                configured: Some(max_linger.as_millis() as u64),
651                startup_measured: None,
652                ongoing_measured: None,
653                effective: eff_linger.as_millis() as u64,
654                reason: if early {
655                    EffectiveReason::Ongoing
656                } else {
657                    EffectiveReason::Configured
658                },
659            },
660            max_linger_ms: max_linger.as_millis() as u64,
661            token_fill_ratio: budget.fill_ratio(),
662            media_ops_total: budget.media_ops_total,
663            overdraft_total: budget.overdraft_total,
664            flushes_total: budget.flushes_total,
665            undersized_deadline_flushes: budget.undersized_deadline_flushes,
666            predicted_media_ops: budget.predicted_media_ops,
667            budget_enabled: budget.config.enabled,
668            budget_mode: budget.config.mode,
669        }
670    }
671}
672
673/// Delete objects under `data_prefix` whose keys are absent from `live`.
674///
675/// See [`LogEngine::reap_orphans`] for safety notes. `live` is typically
676/// [`InMemorySequencer::live_object_ids`](crate::InMemorySequencer::live_object_ids)
677/// or [`ManifestSequencer::live_object_ids`](crate::ManifestSequencer::live_object_ids).
678pub async fn reap_orphans(
679    blob: &dyn BlobStore,
680    data_prefix: &str,
681    live: &HashSet<String>,
682) -> Result<Vec<String>, ObjectLogError> {
683    let keys = blob.list(data_prefix).await?;
684    let mut deleted = Vec::new();
685    for key in keys {
686        if live.contains(&key) {
687            continue;
688        }
689        blob.delete(&key).await?;
690        deleted.push(key);
691    }
692    Ok(deleted)
693}
694
695impl<S: Sequencer> Drop for LogEngine<S> {
696    fn drop(&mut self) {
697        {
698            let mut q = self.shared.queue.lock().expect("poisoned");
699            q.shutdown = true;
700        }
701        self.shared.cv.notify_all();
702        if let Some(t) = self.flush_thread.take() {
703            let _ = t.join();
704        }
705    }
706}
707
708/// Highest numeric suffix already present under `data_prefix`.
709///
710/// Empty `writer_id` reads legacy keys `{prefix}{counter:020}`. A writer id
711/// reads `{prefix}{writer}/{epoch}/{counter}` and ignores other writers, so a
712/// restarted process with a new writer id cannot overwrite their objects.
713async fn recover_data_object_counter(
714    blob: &dyn BlobStore,
715    prefix: &str,
716    writer_id: &str,
717) -> Result<u64, ObjectLogError> {
718    let keys = blob.list(prefix).await?;
719    let mut max = 0u64;
720    let writer_prefix = format!("{prefix}{writer_id}/");
721    for key in keys {
722        let Some(suffix) = key.strip_prefix(prefix) else {
723            continue;
724        };
725        if writer_id.is_empty() {
726            if let Ok(n) = suffix.parse::<u64>() {
727                max = max.max(n);
728            }
729            continue;
730        }
731        let Some(rest) = key.strip_prefix(&writer_prefix) else {
732            continue;
733        };
734        if let Some(n) = rest
735            .rsplit('/')
736            .next()
737            .and_then(|tail| tail.parse::<u64>().ok())
738        {
739            max = max.max(n);
740        }
741    }
742    Ok(max)
743}
744
745fn data_object_key(prefix: &str, writer_id: &str, epoch: u64, counter: u64) -> String {
746    if writer_id.is_empty() {
747        format!("{prefix}{counter:020}")
748    } else {
749        format!("{prefix}{writer_id}/{epoch:020}/{counter:020}")
750    }
751}
752
753fn flush_loop<S>(
754    shared: Arc<Shared<S::Meta>>,
755    blob: Arc<dyn BlobStore>,
756    sequencer: Arc<S>,
757    config: FlushConfig,
758    prefix: String,
759    writer_id: String,
760) where
761    S: Sequencer + 'static,
762    S::Meta: Send + 'static,
763{
764    let max_inflight = config.max_inflight_flushes.max(1);
765    let commit_group_limit = if sequencer.supports_multi_object_commit() {
766        max_inflight
767    } else {
768        1
769    };
770    let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
771        .ok()
772        .and_then(|v| v.parse::<usize>().ok())
773        .filter(|v| *v > 0)
774        .unwrap_or_else(|| max_inflight.min(8));
775    let rt = tokio::runtime::Builder::new_multi_thread()
776        .worker_threads(worker_threads)
777        .enable_all()
778        .build()
779        .expect("flush runtime");
780    // Resume the data-object counter past any keys already under `prefix`. Restarting
781    // at 0 on reopen overwrites sealed objects while manifests still point at the old
782    // byte ranges → RangeOutOfBounds / mid-JSON EOF on fetch (fireweed-481d3e43).
783    let mut counter = match rt.block_on(recover_data_object_counter(
784        blob.as_ref(),
785        &prefix,
786        &writer_id,
787    )) {
788        Ok(counter) => counter,
789        Err(error) => {
790            // A failed listing is not an empty prefix. Fail admission before
791            // any PUT can reuse a sealed object's name.
792            let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
793            return;
794        }
795    };
796    let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
797    let mut active_puts = 0usize;
798    let mut committing: Option<CommitJob> = None;
799    let mut shutdown = false;
800
801    loop {
802        while !shutdown && active_puts < max_inflight {
803            // Poll the commit job even with no queued uploads: it owns bytes
804            // whose release may be needed to admit the next producer.
805            let wait_for_more = pending.is_empty() && committing.is_none();
806            match take_batch(&shared, config, wait_for_more) {
807                TakeBatch::Batch(batch) => {
808                    counter += 1;
809                    let concurrent = max_inflight > 1;
810                    let work = start_flush_work(
811                        &rt, &blob, &prefix, &writer_id, counter, batch, concurrent,
812                    );
813                    if concurrent {
814                        active_puts += 1;
815                    }
816                    pending.push_back(work);
817                }
818                TakeBatch::Empty => break,
819                TakeBatch::Shutdown => shutdown = true,
820            }
821        }
822
823        let mut made_progress = false;
824        for work in pending.iter_mut() {
825            let Some(put) = work.put.as_ref() else {
826                continue;
827            };
828            if !put.is_finished() {
829                continue;
830            }
831            let put = work.put.take().expect("put handle exists");
832            let elapsed = work.put_started.elapsed();
833            work.put_result = Some(match rt.block_on(put) {
834                Ok(Ok(())) => Ok(elapsed),
835                Ok(Err(e)) => Err(e),
836                Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
837                    "flush task failed: {e}"
838                ))),
839            });
840            active_puts = active_puts.saturating_sub(1);
841            made_progress = true;
842        }
843
844        if committing
845            .as_ref()
846            .is_some_and(|job| job.task.is_finished())
847        {
848            let job = committing.take().expect("completed commit exists");
849            match rt.block_on(job.task) {
850                Ok(released) => {
851                    let mut q = shared.queue.lock().expect("poisoned");
852                    q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
853                    shared.cv.notify_all();
854                }
855                Err(error) => {
856                    let uploads = fail_engine_and_take_pending_uploads(
857                        &shared,
858                        &mut pending,
859                        job.first_seq,
860                        ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
861                    );
862                    // A shared runtime can outlive this engine. Await accepted uploads
863                    // explicitly so close/reopen/reaping cannot race orphan writes.
864                    // They are never sequenced after the failed commit.
865                    for upload in uploads {
866                        let _ = rt.block_on(upload);
867                    }
868                    return;
869                }
870            }
871            made_progress = true;
872        }
873
874        if committing.is_none() {
875            if let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit) {
876                if max_inflight == 1 {
877                    // Preserve the low-overhead single-flight path.
878                    let released = finish_flush_work(&shared, &blob, &sequencer, work);
879                    let mut q = shared.queue.lock().expect("poisoned");
880                    q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
881                    shared.cv.notify_all();
882                } else {
883                    // One ordered committer; uploads can progress during its
884                    // durable manifest I/O. Bytes remain charged until completion.
885                    let first_seq = work
886                        .batch
887                        .iter()
888                        .map(|p| p.seq)
889                        .min()
890                        .unwrap_or(work.max_seq);
891                    let shared = shared.clone();
892                    let blob = blob.clone();
893                    let sequencer = sequencer.clone();
894                    committing = Some(CommitJob {
895                        task: rt.spawn_blocking(move || {
896                            finish_flush_work(&shared, &blob, &sequencer, work)
897                        }),
898                        first_seq,
899                    });
900                }
901                made_progress = true;
902            }
903        }
904
905        if pending.is_empty() && committing.is_none() {
906            {
907                let mut q = shared.queue.lock().expect("poisoned");
908                if q.items.is_empty() && q.bytes_in_use == 0 {
909                    // Nothing in flight: any remaining waiters with barrier already
910                    // covered (or no work after last seal) can complete.
911                    notify_flush_waiters(&mut q);
912                }
913            }
914            shared.cv.notify_all();
915            if shutdown {
916                // Fail any remaining flush waiters.
917                let mut q = shared.queue.lock().expect("poisoned");
918                for (_, tx) in q.flush_waiters.drain(..) {
919                    let _ = tx.send(Err(ObjectLogError::Sequencer(
920                        "engine shutting down".into(),
921                    )));
922                }
923                return;
924            }
925            continue;
926        }
927
928        if !made_progress {
929            std::thread::sleep(Duration::from_millis(1));
930        }
931    }
932}
933
934/// Close admission and fail queued callers/barriers after an unrecoverable
935/// startup or commit failure. Return started uploads for explicit draining.
936fn fail_engine_and_take_pending_uploads<M>(
937    shared: &Arc<Shared<M>>,
938    pending: &mut VecDeque<FlushWork<M>>,
939    first_seq: u64,
940    error: ObjectLogError,
941) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
942    let mut uploads = Vec::new();
943    for mut work in pending.drain(..) {
944        if let Some(put) = work.put.take() {
945            uploads.push(put);
946        }
947        send_storage_error(&mut work.responders, error.clone());
948    }
949    // Recover a poisoned guard only to close the engine, never to resume writes.
950    let mut q = shared
951        .queue
952        .lock()
953        .unwrap_or_else(|poisoned| poisoned.into_inner());
954    q.shutdown = true;
955    q.first_failed.get_or_insert((first_seq, error.clone()));
956    for mut item in q.items.drain(..) {
957        if let Some(tx) = item.responder.take() {
958            let _ = tx.send(Err(error.clone()));
959        }
960    }
961    q.bytes = 0;
962    // Keep outstanding bytes conservatively charged until the failed engine
963    // and its accepted uploads have completed.
964    for (_, tx) in q.flush_waiters.drain(..) {
965        let _ = tx.send(Err(error.clone()));
966    }
967    q.force_flush = false;
968    shared.cv.notify_all();
969    uploads
970}
971
972/// Combine only an already-ready contiguous success prefix. Never wait to fill
973/// a group or cross an unfinished/failed PUT; preserve object and batch order.
974fn take_ready_commit_group<M>(
975    pending: &mut VecDeque<FlushWork<M>>,
976    max_objects: usize,
977) -> Option<FlushWork<M>> {
978    pending.front()?.put_result.as_ref()?;
979    let mut group = pending.pop_front().expect("ready front exists");
980    if group.put_result.as_ref().is_some_and(Result::is_err) {
981        return Some(group);
982    }
983    while group.data_objects < max_objects.max(1) {
984        let Some(next) = pending.front() else { break };
985        if !next.put_result.as_ref().is_some_and(Result::is_ok)
986            || group.data_objects.saturating_add(next.data_objects) > max_objects
987        {
988            break;
989        }
990        let mut next = pending.pop_front().expect("ready successor exists");
991        group.batch.append(&mut next.batch);
992        group.locations.append(&mut next.locations);
993        group.responders.append(&mut next.responders);
994        group.bytes = group
995            .bytes
996            .checked_add(next.bytes)
997            .expect("bounded pending bytes");
998        group.max_seq = group.max_seq.max(next.max_seq);
999        group.data_objects += next.data_objects;
1000        let next_elapsed = next.put_result.take().unwrap().unwrap();
1001        let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
1002        *elapsed = (*elapsed).max(next_elapsed);
1003    }
1004    Some(group)
1005}
1006
1007/// How long to sleep before re-evaluating a seal decision.
1008/// - `ZERO` means seal now (early-flush or linger deadline hit).
1009/// - Otherwise wait at most this long (interrupted by new produces).
1010fn effective_linger<M>(
1011    shared: &Shared<M>,
1012    config: FlushConfig,
1013    queued_bytes: usize,
1014    last_enqueue: Option<Instant>,
1015    oldest_enqueue: Option<Instant>,
1016) -> Duration {
1017    if config.linger.is_zero() {
1018        return Duration::ZERO;
1019    }
1020    let now = Instant::now();
1021    if config.budget.enabled {
1022        let mut budget = shared.budget.lock().expect("poisoned");
1023        budget.refill(now);
1024        if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
1025            budget.note_early_flush(now);
1026            return Duration::ZERO;
1027        }
1028    }
1029    // True linger: seal when the oldest buffered item reaches max wait.
1030    if let Some(oldest) = oldest_enqueue {
1031        let deadline = oldest + config.linger;
1032        if now >= deadline {
1033            return Duration::ZERO;
1034        }
1035        let until_deadline = deadline.saturating_duration_since(now);
1036        // Also wake when idle gate could open, so sparse produces don't wait full linger.
1037        if config.budget.enabled
1038            && !config.budget.early_flush_idle.is_zero()
1039            && let Some(last) = last_enqueue
1040        {
1041            let idle_at = last + config.budget.early_flush_idle;
1042            if idle_at > now && idle_at < deadline {
1043                return idle_at.saturating_duration_since(now);
1044            }
1045        }
1046        return until_deadline;
1047    }
1048    config.linger
1049}
1050
1051fn take_batch<M>(
1052    shared: &Arc<Shared<M>>,
1053    config: FlushConfig,
1054    wait_when_empty: bool,
1055) -> TakeBatch<M> {
1056    let mut q = shared.queue.lock().expect("poisoned");
1057    loop {
1058        if q.items.is_empty() {
1059            if q.shutdown {
1060                return TakeBatch::Shutdown;
1061            }
1062            if !wait_when_empty {
1063                return TakeBatch::Empty;
1064            }
1065            // No pending work needs a timer. Enqueue and shutdown change the
1066            // predicate under this mutex and notify. Waiting without
1067            // dropping/reacquiring it first also avoids a lost notification.
1068            q = shared.cv.wait(q).expect("poisoned");
1069            continue;
1070        }
1071
1072        let linger = {
1073            let queued = q.bytes;
1074            let last_enq = q.last_enqueue;
1075            let oldest = q.oldest_enqueue;
1076            drop(q);
1077            let l = effective_linger(shared, config, queued, last_enq, oldest);
1078            q = shared.queue.lock().expect("poisoned");
1079            l
1080        };
1081
1082        let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1083        let force = q.force_flush;
1084        let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1085        if triggered {
1086            if !size_trigger && !q.shutdown && !q.items.is_empty() {
1087                let mut budget = shared.budget.lock().expect("poisoned");
1088                if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1089                    budget.undersized_deadline_flushes += 1;
1090                }
1091            }
1092            break;
1093        }
1094
1095        // Short wait (until idle gate or linger deadline); re-evaluate, do not
1096        // seal solely because a probe sleep timed out.
1097        let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1098        q = guard;
1099    }
1100
1101    let force_drain = q.force_flush;
1102    let mut items = Vec::new();
1103    let mut bytes = 0usize;
1104    // On force_flush, still respect max_bytes so one object stays bounded, but
1105    // ignore max_batches so a barrier can drain large queues across objects.
1106    while let Some(item) = q.items.pop_front() {
1107        q.bytes = q.bytes.saturating_sub(item.payload.len());
1108        bytes += item.payload.len();
1109        items.push(item);
1110        if bytes >= config.max_bytes {
1111            break;
1112        }
1113        if !force_drain && items.len() >= config.max_batches {
1114            break;
1115        }
1116    }
1117    q.oldest_enqueue = if q.items.is_empty() {
1118        None
1119    } else {
1120        // Approximate: next seal window starts now for remaining items.
1121        Some(Instant::now())
1122    };
1123    // Keep forcing while a barrier still has queued work.
1124    if force_drain {
1125        q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1126    }
1127    if items.is_empty() {
1128        TakeBatch::Empty
1129    } else {
1130        TakeBatch::Batch(items)
1131    }
1132}
1133
1134fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1135    let done = q.completed_through;
1136    let mut i = 0;
1137    while i < q.flush_waiters.len() {
1138        let barrier = q.flush_waiters[i].0;
1139        let failure = q
1140            .first_failed
1141            .as_ref()
1142            .filter(|(sequence, _)| *sequence <= barrier)
1143            .map(|(_, error)| error.clone());
1144        if barrier <= done || failure.is_some() {
1145            let (_, tx) = q.flush_waiters.swap_remove(i);
1146            let _ = tx.send(failure.map_or(Ok(()), Err));
1147        } else {
1148            i += 1;
1149        }
1150    }
1151    if q.flush_waiters.is_empty() {
1152        q.force_flush = false;
1153    } else if q.items.is_empty() && q.bytes_in_use == 0 {
1154        // Barriers still waiting but no work — should not happen; clear force.
1155        q.force_flush = true;
1156    } else {
1157        q.force_flush = true;
1158    }
1159}
1160
1161fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1162    for (_, tx) in responders.iter_mut() {
1163        if let Some(tx) = tx.take() {
1164            let _ = tx.send(Err(err.clone()));
1165        }
1166    }
1167}
1168
1169fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1170    for (durability, tx) in responders.iter_mut() {
1171        if *durability == Durability::Durable
1172            && let Some(tx) = tx.take()
1173        {
1174            let _ = tx.send(Ok(AppendOutcome {
1175                base_offset: None,
1176                last_offset: None,
1177                durable: true,
1178                sequenced: false,
1179            }));
1180        }
1181    }
1182}
1183
1184fn prepare_flush_work<M>(
1185    prefix: &str,
1186    writer_id: &str,
1187    counter: u64,
1188    mut batch: Vec<Pending<M>>,
1189) -> (FlushWork<M>, String, Vec<Bytes>) {
1190    let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1191    let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1192    let epoch = batch.first().map(|pending| pending.epoch).unwrap_or(0);
1193    let key = data_object_key(prefix, writer_id, epoch, counter);
1194    let mut offset = 0usize;
1195    for p in &batch {
1196        let start = offset as u32;
1197        offset += p.payload.len();
1198        chunks.push(p.payload.clone());
1199        locations.push(BatchLocation {
1200            object_id: key.clone(),
1201            byte_start: start,
1202            byte_len: p.payload.len() as u32,
1203        });
1204    }
1205    if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1206        eprintln!(
1207            "object-log seal: key={key} batches={} bytes={offset}",
1208            batch.len()
1209        );
1210    }
1211    let responders: Vec<(Durability, Option<Responder>)> = batch
1212        .iter_mut()
1213        .map(|p| (p.durability, p.responder.take()))
1214        .collect();
1215    let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1216    let work = FlushWork {
1217        batch,
1218        locations,
1219        responders,
1220        bytes: offset,
1221        max_seq,
1222        data_objects: 1,
1223        put: None,
1224        put_started: Instant::now(),
1225        put_result: None,
1226    };
1227    (work, key, chunks)
1228}
1229
1230/// Start a durable put for a sealed batch.
1231///
1232/// - `concurrent == false` (default `max_inflight_flushes == 1`): `block_on` the
1233///   put on the flush thread; Local uses `block_in_place` (no spawn_blocking queue).
1234/// - `concurrent == true`: spawn put tasks for parallel S3-style throughput.
1235fn start_flush_work<M>(
1236    rt: &tokio::runtime::Runtime,
1237    blob: &Arc<dyn BlobStore>,
1238    prefix: &str,
1239    writer_id: &str,
1240    counter: u64,
1241    batch: Vec<Pending<M>>,
1242    concurrent: bool,
1243) -> FlushWork<M> {
1244    let (mut work, key, chunks) = prepare_flush_work(prefix, writer_id, counter, batch);
1245    let blob = Arc::clone(blob);
1246    let _ = blob.take_media_op_stats();
1247    work.put_started = Instant::now();
1248    if concurrent {
1249        work.put =
1250            Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
1251    } else {
1252        let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1253        let elapsed = work.put_started.elapsed();
1254        work.put_result = Some(match put_result {
1255            Ok(()) => Ok(elapsed),
1256            Err(e) => Err(e),
1257        });
1258    }
1259    work
1260}
1261
1262fn finish_flush_work<S, M>(
1263    shared: &Arc<Shared<M>>,
1264    blob: &Arc<dyn BlobStore>,
1265    sequencer: &Arc<S>,
1266    mut work: FlushWork<S::Meta>,
1267) -> usize
1268where
1269    S: Sequencer + 'static,
1270    S::Meta: Send + 'static,
1271{
1272    let release_bytes = work.bytes;
1273    let first_seq = work
1274        .batch
1275        .iter()
1276        .map(|pending| pending.seq)
1277        .min()
1278        .unwrap_or(work.max_seq);
1279    // Durable-then-sequence: the object PUT may have overlapped later PUTs, but
1280    // sequencer commits are still completed in object creation order.
1281    // Grouped put_ms is the largest observed member PUT duration, not a sum.
1282    let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1283    let put_elapsed = match work.put_result.take().expect("put result is ready") {
1284        Ok(elapsed) => elapsed,
1285        Err(e) => {
1286            send_storage_error(&mut work.responders, e.clone());
1287            let mut q = shared.queue.lock().expect("poisoned");
1288            // Unblock flush waiters covering this seq with the storage error.
1289            let done = work.max_seq;
1290            if done >= q.completed_through {
1291                q.completed_through = done;
1292            }
1293            q.first_failed.get_or_insert((first_seq, e));
1294            notify_flush_waiters(&mut q);
1295            shared.cv.notify_all();
1296            return release_bytes;
1297        }
1298    };
1299
1300    // Media ops for the data object put.
1301    let mut media_ops = blob
1302        .take_media_op_stats()
1303        .map(|s| s.media_ops)
1304        .unwrap_or(work.data_objects as u64); // fallback: 1 per successful put
1305
1306    // Signal Durable-level waiters now (after PUT, before commit).
1307    send_durable_acks(&mut work.responders);
1308
1309    // Sequence the ready group atomically, preserving each object's locations.
1310    let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1311        .batch
1312        .iter()
1313        .zip(work.locations.iter())
1314        .map(|(p, loc)| CommitBatch {
1315            partition: p.partition.clone(),
1316            record_count: p.record_count,
1317            location: loc.clone(),
1318            meta: &p.meta,
1319            epoch: p.epoch,
1320        })
1321        .collect();
1322
1323    let commit_started = timing.then(Instant::now);
1324    // Clear stats again so sequencer durable work (e.g. ManifestSequencer put)
1325    // on the shared store is counted into the same budget.
1326    let _ = blob.take_media_op_stats();
1327    let mut commit_error = None;
1328    match sequencer.commit(&commit_batches) {
1329        Ok(outcomes) => {
1330            if let Some(commit_started) = commit_started {
1331                eprintln!(
1332                    "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1333                    work.bytes,
1334                    commit_batches.len(),
1335                    put_elapsed.as_millis(),
1336                    commit_started.elapsed().as_millis(),
1337                    work.data_objects
1338                );
1339            }
1340            if let Some(s) = blob.take_media_op_stats() {
1341                media_ops = media_ops.saturating_add(s.media_ops);
1342            }
1343            for ((outcome, (_, tx)), pending) in outcomes
1344                .into_iter()
1345                .zip(work.responders.iter_mut())
1346                .zip(work.batch.iter())
1347            {
1348                if let Some(tx) = tx.take() {
1349                    let (base, last) = match outcome {
1350                        CommitOutcome::Assigned {
1351                            base_offset,
1352                            record_count,
1353                        } => (
1354                            Some(base_offset),
1355                            Some(base_offset + record_count as i64 - 1),
1356                        ),
1357                        CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1358                        CommitOutcome::Rejected { reason } => {
1359                            // A partition fenced past this batch's epoch will never accept it:
1360                            // report that as a definite rejection, not a generic sequencer error.
1361                            let error = match sequencer.partition_epoch(&pending.partition) {
1362                                Some(current) if current > pending.epoch => {
1363                                    ObjectLogError::Fenced {
1364                                        partition: pending.partition.as_str().to_owned(),
1365                                        epoch: pending.epoch,
1366                                        current,
1367                                    }
1368                                }
1369                                _ => ObjectLogError::Sequencer(reason),
1370                            };
1371                            let _ = tx.send(Err(error));
1372                            continue;
1373                        }
1374                    };
1375                    let _ = tx.send(Ok(AppendOutcome {
1376                        base_offset: base,
1377                        last_offset: last,
1378                        durable: true,
1379                        sequenced: true,
1380                    }));
1381                }
1382            }
1383        }
1384        Err(e) => {
1385            if let Some(s) = blob.take_media_op_stats() {
1386                media_ops = media_ops.saturating_add(s.media_ops);
1387            }
1388            send_storage_error(&mut work.responders, e.clone());
1389            commit_error = Some(e);
1390        }
1391    }
1392
1393    {
1394        let mut budget = shared.budget.lock().expect("poisoned");
1395        budget.consume_after_flush(media_ops, Instant::now());
1396    }
1397    {
1398        let mut q = shared.queue.lock().expect("poisoned");
1399        if let Some(error) = commit_error {
1400            q.first_failed.get_or_insert((first_seq, error));
1401        }
1402        if work.max_seq >= q.completed_through {
1403            q.completed_through = work.max_seq;
1404        }
1405        notify_flush_waiters(&mut q);
1406    }
1407    shared.cv.notify_all();
1408
1409    release_bytes
1410}
1411
1412#[cfg(test)]
1413mod flush_barrier_tests {
1414    use super::*;
1415
1416    #[test]
1417    fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1418        let error = ObjectLogError::Sequencer("manifest failed".into());
1419        let mut queue = Queue::<()> {
1420            items: VecDeque::new(),
1421            bytes: 0,
1422            bytes_in_use: 1,
1423            shutdown: false,
1424            next_seq: 4,
1425            completed_through: 2,
1426            first_failed: Some((2, error.clone())),
1427            force_flush: true,
1428            flush_waiters: Vec::new(),
1429            last_enqueue: None,
1430            oldest_enqueue: None,
1431        };
1432        let mut receivers = Vec::new();
1433        for barrier in [1, 2, 3] {
1434            let (tx, rx) = oneshot::channel();
1435            queue.flush_waiters.push((barrier, tx));
1436            receivers.push(rx);
1437        }
1438        notify_flush_waiters(&mut queue);
1439        assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1440        assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1441        assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1442        assert!(queue.flush_waiters.is_empty());
1443    }
1444}
1445
1446#[cfg(test)]
1447mod ready_commit_group_tests {
1448    use super::*;
1449    use crate::{ManifestSequencer, MemoryBlobStore};
1450    use async_trait::async_trait;
1451    use std::ops::Range;
1452
1453    fn work(
1454        n: u64,
1455        durability: Durability,
1456    ) -> (
1457        FlushWork<()>,
1458        String,
1459        Vec<Bytes>,
1460        oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1461    ) {
1462        let (tx, rx) = oneshot::channel();
1463        let (mut work, key, chunks) = prepare_flush_work(
1464            "data/",
1465            "",
1466            n,
1467            vec![Pending {
1468                partition: PartitionKey("shared".into()),
1469                record_count: 1,
1470                payload: Bytes::from(format!("body-{n}")),
1471                meta: (),
1472                epoch: 0,
1473                durability,
1474                responder: Some(tx),
1475                seq: n,
1476            }],
1477        );
1478        work.put_result = Some(Ok(Duration::from_millis(n)));
1479        (work, key, chunks, rx)
1480    }
1481
1482    #[test]
1483    fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1484        let mut pending = (1..=5)
1485            .map(|n| work(n, Durability::Sequenced).0)
1486            .collect::<VecDeque<_>>();
1487        let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1488        let mut released = 0;
1489        for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1490            let group = take_ready_commit_group(&mut pending, 2).unwrap();
1491            assert_eq!(
1492                group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1493                expected
1494            );
1495            assert_eq!(group.data_objects, expected.len());
1496            assert_eq!(group.responders.len(), expected.len());
1497            assert_eq!(group.max_seq, *expected.last().unwrap());
1498            for (location, n) in group.locations.iter().zip(&expected) {
1499                assert_eq!(location.object_id, format!("data/{n:020}"));
1500                assert_eq!(location.byte_start, 0);
1501                assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1502            }
1503            released += group.bytes;
1504        }
1505        assert!(pending.is_empty());
1506        assert_eq!(released, original_bytes);
1507
1508        let mut first = work(1, Durability::Sequenced).0;
1509        first.put_result = None;
1510        let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1511        assert!(take_ready_commit_group(&mut pending, 8).is_none());
1512        assert_eq!(
1513            pending.len(),
1514            2,
1515            "a later ready PUT cannot pass an unfinished head"
1516        );
1517        pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1518        let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1519        assert_eq!(failed.data_objects, 1);
1520        assert!(failed.put_result.unwrap().is_err());
1521        assert_eq!(pending[0].max_seq, 2);
1522
1523        let mut failed = work(2, Durability::Sequenced).0;
1524        failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1525        let mut pending = VecDeque::from([
1526            work(1, Durability::Sequenced).0,
1527            failed,
1528            work(3, Durability::Sequenced).0,
1529        ]);
1530        assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1531        assert_eq!(
1532            pending.len(),
1533            2,
1534            "a success group cannot cross a failed PUT"
1535        );
1536        let mut pending = VecDeque::from([
1537            work(1, Durability::Sequenced).0,
1538            work(2, Durability::Sequenced).0,
1539        ]);
1540        assert_eq!(
1541            take_ready_commit_group(&mut pending, 1)
1542                .unwrap()
1543                .data_objects,
1544            1
1545        );
1546        assert_eq!(
1547            pending.len(),
1548            1,
1549            "one-object sequencers retain their call boundary"
1550        );
1551    }
1552
1553    struct GatedManifestStore {
1554        inner: MemoryBlobStore,
1555        entered: std::sync::mpsc::SyncSender<()>,
1556        release: Mutex<Option<oneshot::Receiver<()>>>,
1557        fail: bool,
1558        panic_on_commit: bool,
1559    }
1560
1561    #[async_trait]
1562    impl BlobStore for GatedManifestStore {
1563        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1564            if key.starts_with("manifest/") {
1565                let release = self.release.lock().unwrap().take();
1566                if let Some(release) = release {
1567                    self.entered.send(()).unwrap();
1568                    release.await.unwrap();
1569                    assert!(!self.panic_on_commit, "injected commit panic");
1570                    if self.fail {
1571                        return Err(ObjectLogError::StorageUnavailable(
1572                            "manifest failure".into(),
1573                        ));
1574                    }
1575                }
1576            }
1577            self.inner.put(key, value).await
1578        }
1579        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1580            self.inner.get(key).await
1581        }
1582        async fn get_range(
1583            &self,
1584            key: &str,
1585            range: Range<u64>,
1586        ) -> Result<Option<Bytes>, ObjectLogError> {
1587            self.inner.get_range(key, range).await
1588        }
1589        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1590            self.inner.list(prefix).await
1591        }
1592        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1593            self.inner.delete(key).await
1594        }
1595        fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1596            self.inner.take_media_op_stats()
1597        }
1598    }
1599
1600    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1601    async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1602        for fail in [false, true] {
1603            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1604            let (release_tx, release_rx) = oneshot::channel();
1605            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1606                inner: MemoryBlobStore::new(),
1607                entered: entered_tx,
1608                release: Mutex::new(Some(release_rx)),
1609                fail,
1610                panic_on_commit: false,
1611            });
1612            let sequencer = Arc::new(
1613                ManifestSequencer::open(blob.clone(), "manifest/")
1614                    .await
1615                    .unwrap(),
1616            );
1617            assert!(sequencer.supports_multi_object_commit());
1618            let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1619            let (mut second, second_key, second_chunks, mut durable_rx) =
1620                work(2, Durability::Durable);
1621            second.batch[0].partition = PartitionKey("other".into());
1622            let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1623            blob.put_chunks(&first_key, first_chunks).await.unwrap();
1624            blob.put_chunks(&second_key, second_chunks).await.unwrap();
1625            blob.put_chunks(&third_key, third_chunks).await.unwrap();
1626            let mut pending = VecDeque::from([first, second, third]);
1627            let group = take_ready_commit_group(&mut pending, 8).unwrap();
1628            let bytes = group.bytes;
1629            assert_eq!(group.data_objects, 3);
1630            let (flush_tx, mut flush_rx) = oneshot::channel();
1631            let mut config = FlushConfig::default();
1632            config.budget.enabled = false;
1633            let shared: Arc<Shared<()>> = Arc::new(Shared {
1634                queue: Mutex::new(Queue {
1635                    items: VecDeque::new(),
1636                    bytes: 0,
1637                    bytes_in_use: bytes,
1638                    shutdown: false,
1639                    next_seq: 4,
1640                    completed_through: 0,
1641                    first_failed: None,
1642                    force_flush: true,
1643                    flush_waiters: vec![(3, flush_tx)],
1644                    last_enqueue: None,
1645                    oldest_enqueue: None,
1646                }),
1647                cv: Condvar::new(),
1648                max_buffered_bytes: bytes,
1649                budget: Mutex::new(BudgetRuntime::new(config.budget)),
1650                flush_config: config,
1651            });
1652            let worker = {
1653                let shared = shared.clone();
1654                let blob = blob.clone();
1655                let sequencer = sequencer.clone();
1656                std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1657            };
1658            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1659            assert!(matches!(
1660                sequenced_rx.try_recv(),
1661                Err(oneshot::error::TryRecvError::Empty)
1662            ));
1663            assert!(matches!(
1664                third_rx.try_recv(),
1665                Err(oneshot::error::TryRecvError::Empty)
1666            ));
1667            assert!(matches!(
1668                flush_rx.try_recv(),
1669                Err(oneshot::error::TryRecvError::Empty)
1670            ));
1671            let durable = durable_rx.try_recv().unwrap().unwrap();
1672            assert!(durable.durable && !durable.sequenced);
1673            release_tx.send(()).unwrap();
1674            assert_eq!(worker.join().unwrap(), bytes);
1675            if fail {
1676                assert!(sequenced_rx.await.unwrap().is_err());
1677                assert!(third_rx.await.unwrap().is_err());
1678                assert!(flush_rx.await.unwrap().is_err());
1679                assert_eq!(
1680                    sequencer
1681                        .high_watermark(&PartitionKey("shared".into()))
1682                        .unwrap(),
1683                    0
1684                );
1685                assert!(blob.list("manifest/").await.unwrap().is_empty());
1686            } else {
1687                assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1688                assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1689                flush_rx.await.unwrap().unwrap();
1690                assert_eq!(blob.list("manifest/").await.unwrap().len(), 3);
1691            }
1692            assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1693            let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1694                .await
1695                .unwrap();
1696            for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1697                let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1698                assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1699                for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1700                    assert_eq!(entry.base_offset, offset as i64);
1701                    let loc = &entry.location;
1702                    assert_eq!(loc.object_id, format!("data/{object:020}"));
1703                    let body = blob
1704                        .get_range(
1705                            &loc.object_id,
1706                            loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1707                        )
1708                        .await
1709                        .unwrap()
1710                        .unwrap();
1711                    assert_eq!(body, format!("body-{object}"));
1712                }
1713            }
1714        }
1715    }
1716
1717    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1718    async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1719        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1720        let (release_tx, release_rx) = oneshot::channel();
1721        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1722            inner: MemoryBlobStore::new(),
1723            entered: entered_tx,
1724            release: Mutex::new(Some(release_rx)),
1725            fail: false,
1726            panic_on_commit: false,
1727        });
1728        let sequencer = Arc::new(
1729            ManifestSequencer::open(blob.clone(), "manifest/")
1730                .await
1731                .unwrap(),
1732        );
1733        let mut config = FlushConfig::default();
1734        config.max_batches = 1;
1735        config.max_inflight_flushes = 4;
1736        config.linger = Duration::ZERO;
1737        config.budget.enabled = false;
1738        let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1739        let first = {
1740            let engine = engine.clone();
1741            tokio::spawn(async move {
1742                engine
1743                    .produce(
1744                        PartitionKey("p".into()),
1745                        Bytes::from_static(b"first"),
1746                        1,
1747                        (),
1748                        Durability::Sequenced,
1749                    )
1750                    .await
1751            })
1752        };
1753        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1754        for n in 1..4 {
1755            engine
1756                .produce(
1757                    PartitionKey("p".into()),
1758                    Bytes::from(format!("later-{n}")),
1759                    1,
1760                    (),
1761                    Durability::Buffered,
1762                )
1763                .await
1764                .unwrap();
1765        }
1766        let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1767            loop {
1768                if blob.list("data/").await.unwrap().len() == 4 {
1769                    break;
1770                }
1771                tokio::time::sleep(Duration::from_millis(1)).await;
1772            }
1773        })
1774        .await
1775        .is_ok();
1776        let early_ack = first.is_finished();
1777        // Always release before assertions so a failed regression can drain.
1778        release_tx.send(()).unwrap();
1779        first.await.unwrap().unwrap();
1780        engine.flush().await.unwrap();
1781        let rows = engine
1782            .fetch(&PartitionKey("p".into()), 0, 1024)
1783            .await
1784            .unwrap();
1785        assert_eq!(
1786            rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
1787            vec![0, 1, 2, 3]
1788        );
1789        drop(engine);
1790        assert!(
1791            !early_ack,
1792            "sequenced acknowledgement preceded manifest durability"
1793        );
1794        assert!(
1795            upload_progress,
1796            "blocked manifest prevented independent data uploads"
1797        );
1798    }
1799
1800    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1801    async fn commit_completion_releases_admission_bytes_without_more_uploads() {
1802        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1803        let sequencer = Arc::new(
1804            ManifestSequencer::open(blob.clone(), "manifest/")
1805                .await
1806                .unwrap(),
1807        );
1808        let mut config = FlushConfig::default();
1809        config.max_bytes = 4;
1810        config.max_buffered_bytes = 4;
1811        config.max_batches = 1;
1812        config.max_inflight_flushes = 4;
1813        config.linger = Duration::ZERO;
1814        config.budget.enabled = false;
1815        let engine = LogEngine::new(blob, sequencer, config, "data/");
1816        engine
1817            .produce(
1818                PartitionKey("p".into()),
1819                Bytes::from_static(b"full"),
1820                1,
1821                (),
1822                Durability::Sequenced,
1823            )
1824            .await
1825            .unwrap();
1826        tokio::time::timeout(Duration::from_secs(5), async {
1827            while engine.buffer_stats().bytes_in_use != 0 {
1828                tokio::time::sleep(Duration::from_millis(1)).await;
1829            }
1830        })
1831        .await
1832        .expect("completed committer stranded admission bytes");
1833        engine
1834            .produce(
1835                PartitionKey("p".into()),
1836                Bytes::from_static(b"next"),
1837                1,
1838                (),
1839                Durability::Sequenced,
1840            )
1841            .await
1842            .unwrap();
1843        engine.flush().await.unwrap();
1844        assert_eq!(
1845            engine
1846                .fetch(&PartitionKey("p".into()), 0, 100)
1847                .await
1848                .unwrap()
1849                .len(),
1850            2
1851        );
1852    }
1853
1854    struct FailureInjectingStore {
1855        inner: Arc<dyn BlobStore>,
1856        second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
1857        second_upload_completed: std::sync::atomic::AtomicBool,
1858        fail_data_listing: std::sync::atomic::AtomicBool,
1859    }
1860
1861    impl FailureInjectingStore {
1862        fn new(inner: Arc<dyn BlobStore>) -> Self {
1863            Self {
1864                inner,
1865                second_upload_gate: Mutex::new(None),
1866                second_upload_completed: std::sync::atomic::AtomicBool::new(false),
1867                fail_data_listing: std::sync::atomic::AtomicBool::new(false),
1868            }
1869        }
1870    }
1871
1872    #[async_trait]
1873    impl BlobStore for FailureInjectingStore {
1874        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1875            let gate = if key == "data/00000000000000000002" {
1876                self.second_upload_gate.lock().unwrap().take()
1877            } else {
1878                None
1879            };
1880            if let Some((entered, release)) = gate {
1881                entered.send(()).unwrap();
1882                release.await.unwrap();
1883                let result = self.inner.put(key, value).await;
1884                if result.is_ok() {
1885                    self.second_upload_completed
1886                        .store(true, std::sync::atomic::Ordering::SeqCst);
1887                }
1888                return result;
1889            }
1890            self.inner.put(key, value).await
1891        }
1892        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1893            self.inner.get(key).await
1894        }
1895        async fn get_range(
1896            &self,
1897            key: &str,
1898            range: Range<u64>,
1899        ) -> Result<Option<Bytes>, ObjectLogError> {
1900            self.inner.get_range(key, range).await
1901        }
1902        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1903            if prefix == "data/"
1904                && self
1905                    .fail_data_listing
1906                    .load(std::sync::atomic::Ordering::SeqCst)
1907            {
1908                return Err(ObjectLogError::StorageUnavailable(
1909                    "injected object listing failure".into(),
1910                ));
1911            }
1912            self.inner.list(prefix).await
1913        }
1914        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1915            self.inner.delete(key).await
1916        }
1917    }
1918
1919    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1920    async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
1921        let inner = Arc::new(MemoryBlobStore::new());
1922        let key = "data/00000000000000000001";
1923        let original = Bytes::from_static(b"existing immutable object");
1924        inner.put(key, original.clone()).await.unwrap();
1925        let store = Arc::new(FailureInjectingStore::new(inner.clone()));
1926        store
1927            .fail_data_listing
1928            .store(true, std::sync::atomic::Ordering::SeqCst);
1929        let mut config = FlushConfig::default();
1930        config.max_batches = 1;
1931        config.max_inflight_flushes = 4;
1932        config.linger = Duration::ZERO;
1933        config.budget.enabled = false;
1934        let engine = LogEngine::new(
1935            store,
1936            Arc::new(crate::InMemorySequencer::new()),
1937            config,
1938            "data/",
1939        );
1940        let result = tokio::time::timeout(
1941            Duration::from_secs(5),
1942            engine.produce(
1943                PartitionKey("p".into()),
1944                Bytes::from_static(b"must not overwrite"),
1945                1,
1946                (),
1947                Durability::Sequenced,
1948            ),
1949        )
1950        .await;
1951        let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
1952        tokio::task::spawn_blocking(move || drop(engine))
1953            .await
1954            .unwrap();
1955        let retained = inner.get(key).await.unwrap();
1956        assert_eq!(
1957            retained,
1958            Some(original),
1959            "failed listing must preserve sealed bytes"
1960        );
1961        assert!(
1962            result.unwrap().is_err(),
1963            "failed prefix recovery must reject admission"
1964        );
1965        assert!(
1966            flushed.unwrap().is_err(),
1967            "flush must report failed initialization"
1968        );
1969        assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
1970    }
1971
1972    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1973    async fn engines_commit_and_close_independently() {
1974        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1975        let (release_tx, release_rx) = oneshot::channel();
1976        let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1977            inner: MemoryBlobStore::new(),
1978            entered: entered_tx,
1979            release: Mutex::new(Some(release_rx)),
1980            fail: false,
1981            panic_on_commit: false,
1982        })));
1983        let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
1984        let mut config = FlushConfig::default();
1985        config.max_batches = 1;
1986        config.max_inflight_flushes = 4;
1987        config.linger = Duration::ZERO;
1988        config.budget.enabled = false;
1989        let a_seq = Arc::new(
1990            ManifestSequencer::open(a_blob.clone(), "manifest/")
1991                .await
1992                .unwrap(),
1993        );
1994        let b_seq = Arc::new(
1995            ManifestSequencer::open(b_blob.clone(), "manifest/")
1996                .await
1997                .unwrap(),
1998        );
1999        let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
2000        a.produce(
2001            PartitionKey("p".into()),
2002            Bytes::from_static(b"a-first"),
2003            1,
2004            (),
2005            Durability::Buffered,
2006        )
2007        .await
2008        .unwrap();
2009        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2010        let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
2011        let b_result = tokio::time::timeout(
2012            Duration::from_secs(5),
2013            b.produce(
2014                PartitionKey("p".into()),
2015                Bytes::from_static(b"b-first"),
2016                1,
2017                (),
2018                Durability::Sequenced,
2019            ),
2020        )
2021        .await;
2022        let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
2023        let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
2024        // Always release the held manifest before assertions, including the pre-change red run.
2025        release_tx.send(()).unwrap();
2026        if b_closed.is_err() {
2027            closing_b.await.unwrap();
2028        }
2029        assert!(
2030            b_closed.is_ok(),
2031            "closing B waited for A's blocked manifest"
2032        );
2033        assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
2034        a.flush().await.unwrap();
2035        assert_eq!(
2036            a.produce(
2037                PartitionKey("p".into()),
2038                Bytes::from_static(b"a-second"),
2039                1,
2040                (),
2041                Durability::Sequenced
2042            )
2043            .await
2044            .unwrap()
2045            .base_offset,
2046            Some(1)
2047        );
2048        let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
2049        assert_eq!(
2050            rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
2051            vec![
2052                Bytes::from_static(b"a-first"),
2053                Bytes::from_static(b"a-second")
2054            ]
2055        );
2056        tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
2057    }
2058
2059    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2060    async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
2061        let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
2062        let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
2063        let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
2064        let (upload_release_tx, upload_release_rx) = oneshot::channel();
2065        let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2066            inner: MemoryBlobStore::new(),
2067            entered: manifest_entered_tx,
2068            release: Mutex::new(Some(manifest_release_rx)),
2069            fail: false,
2070            panic_on_commit: true,
2071        })));
2072        *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
2073        let mut config = FlushConfig::default();
2074        config.max_batches = 1;
2075        config.max_inflight_flushes = 4;
2076        config.linger = Duration::ZERO;
2077        config.budget.enabled = false;
2078        let sequencer = Arc::new(
2079            ManifestSequencer::open(store.clone(), "manifest/")
2080                .await
2081                .unwrap(),
2082        );
2083        let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
2084        engine
2085            .produce(
2086                PartitionKey("p".into()),
2087                Bytes::from_static(b"first"),
2088                1,
2089                (),
2090                Durability::Buffered,
2091            )
2092            .await
2093            .unwrap();
2094        manifest_entered_rx
2095            .recv_timeout(Duration::from_secs(5))
2096            .unwrap();
2097        engine
2098            .produce(
2099                PartitionKey("p".into()),
2100                Bytes::from_static(b"second"),
2101                1,
2102                (),
2103                Durability::Buffered,
2104            )
2105            .await
2106            .unwrap();
2107        upload_entered_rx
2108            .recv_timeout(Duration::from_secs(5))
2109            .unwrap();
2110        manifest_release_tx.send(()).unwrap();
2111        tokio::time::timeout(Duration::from_secs(5), async {
2112            while !engine.shared.queue.lock().unwrap().shutdown {
2113                tokio::time::sleep(Duration::from_millis(1)).await;
2114            }
2115        })
2116        .await
2117        .unwrap();
2118        let flush_failed = engine.flush().await.is_err();
2119        // A failed committer closes admission first; its accepted PUT must still
2120        // complete before close returns, even though no manifest may reference it.
2121        let upload_was_owned = upload_release_tx.send(()).is_ok();
2122        tokio::time::timeout(
2123            Duration::from_secs(5),
2124            tokio::task::spawn_blocking(move || drop(engine)),
2125        )
2126        .await
2127        .unwrap()
2128        .unwrap();
2129        assert!(flush_failed);
2130        assert!(upload_was_owned);
2131        assert!(
2132            store
2133                .second_upload_completed
2134                .load(std::sync::atomic::Ordering::SeqCst)
2135        );
2136        assert_eq!(store.list("data/").await.unwrap().len(), 2);
2137        let sequencer = Arc::new(
2138            ManifestSequencer::open(store.clone(), "manifest/")
2139                .await
2140                .unwrap(),
2141        );
2142        let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2143        assert!(
2144            reopened
2145                .fetch(&PartitionKey("p".into()), 0, 100)
2146                .await
2147                .unwrap()
2148                .is_empty()
2149        );
2150        assert_eq!(
2151            reopened
2152                .produce(
2153                    PartitionKey("p".into()),
2154                    Bytes::from_static(b"after-reopen"),
2155                    1,
2156                    (),
2157                    Durability::Sequenced
2158                )
2159                .await
2160                .unwrap()
2161                .base_offset,
2162            Some(0)
2163        );
2164        let rows = reopened
2165            .fetch(&PartitionKey("p".into()), 0, 100)
2166            .await
2167            .unwrap();
2168        assert_eq!(rows.len(), 1);
2169        assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2170        assert_eq!(store.list("data/").await.unwrap().len(), 3);
2171        tokio::task::spawn_blocking(move || drop(reopened))
2172            .await
2173            .unwrap();
2174    }
2175
2176    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2177    async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2178        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2179        let (release_tx, release_rx) = oneshot::channel();
2180        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2181            inner: MemoryBlobStore::new(),
2182            entered: entered_tx,
2183            release: Mutex::new(Some(release_rx)),
2184            fail: false,
2185            panic_on_commit: false,
2186        });
2187        let sequencer = Arc::new(
2188            ManifestSequencer::open(blob.clone(), "manifest/")
2189                .await
2190                .unwrap(),
2191        );
2192        let mut config = FlushConfig::default();
2193        config.max_batches = 1;
2194        config.max_inflight_flushes = 4;
2195        config.linger = Duration::ZERO;
2196        config.budget.enabled = false;
2197        let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2198        engine
2199            .produce(
2200                PartitionKey("p".into()),
2201                Bytes::from_static(b"first"),
2202                1,
2203                (),
2204                Durability::Buffered,
2205            )
2206            .await
2207            .unwrap();
2208        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2209        engine
2210            .produce(
2211                PartitionKey("p".into()),
2212                Bytes::from_static(b"second"),
2213                1,
2214                (),
2215                Durability::Buffered,
2216            )
2217            .await
2218            .unwrap();
2219        let shared = engine.shared.clone();
2220        let dropping = tokio::task::spawn_blocking(move || drop(engine));
2221        tokio::time::timeout(Duration::from_secs(5), async {
2222            while !shared.queue.lock().unwrap().shutdown {
2223                tokio::time::sleep(Duration::from_millis(1)).await;
2224            }
2225        })
2226        .await
2227        .unwrap();
2228        let premature = dropping.is_finished();
2229        release_tx.send(()).unwrap();
2230        tokio::time::timeout(Duration::from_secs(5), dropping)
2231            .await
2232            .unwrap()
2233            .unwrap();
2234        assert!(!premature, "shutdown returned before manifest durability");
2235        let reopened = Arc::new(
2236            ManifestSequencer::open(blob.clone(), "manifest/")
2237                .await
2238                .unwrap(),
2239        );
2240        let engine = LogEngine::new(blob, reopened, config, "data/");
2241        let rows = engine
2242            .fetch(&PartitionKey("p".into()), 0, 100)
2243            .await
2244            .unwrap();
2245        assert_eq!(
2246            rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2247            vec![b"first".as_slice(), b"second".as_slice()]
2248        );
2249    }
2250
2251    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2252    async fn panicked_committer_closes_admission_and_fails_flush() {
2253        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2254        let (release_tx, release_rx) = oneshot::channel();
2255        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2256            inner: MemoryBlobStore::new(),
2257            entered: entered_tx,
2258            release: Mutex::new(Some(release_rx)),
2259            fail: false,
2260            panic_on_commit: true,
2261        });
2262        let sequencer = Arc::new(
2263            ManifestSequencer::open(blob.clone(), "manifest/")
2264                .await
2265                .unwrap(),
2266        );
2267        let mut config = FlushConfig::default();
2268        config.max_batches = 1;
2269        config.max_inflight_flushes = 4;
2270        config.linger = Duration::ZERO;
2271        config.budget.enabled = false;
2272        let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2273        let first = {
2274            let engine = engine.clone();
2275            tokio::spawn(async move {
2276                engine
2277                    .produce(
2278                        PartitionKey("p".into()),
2279                        Bytes::from_static(b"first"),
2280                        1,
2281                        (),
2282                        Durability::Sequenced,
2283                    )
2284                    .await
2285            })
2286        };
2287        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2288        engine
2289            .produce(
2290                PartitionKey("p".into()),
2291                Bytes::from_static(b"later"),
2292                1,
2293                (),
2294                Durability::Buffered,
2295            )
2296            .await
2297            .unwrap();
2298        let flushing = {
2299            let engine = engine.clone();
2300            tokio::spawn(async move { engine.flush().await })
2301        };
2302        release_tx.send(()).unwrap();
2303        assert!(
2304            tokio::time::timeout(Duration::from_secs(5), first)
2305                .await
2306                .unwrap()
2307                .unwrap()
2308                .is_err()
2309        );
2310        assert!(
2311            tokio::time::timeout(Duration::from_secs(5), flushing)
2312                .await
2313                .unwrap()
2314                .unwrap()
2315                .is_err()
2316        );
2317        tokio::time::timeout(Duration::from_secs(5), async {
2318            while !engine.shared.queue.lock().unwrap().shutdown {
2319                tokio::time::sleep(Duration::from_millis(1)).await;
2320            }
2321        })
2322        .await
2323        .unwrap();
2324        assert!(
2325            engine
2326                .produce(
2327                    PartitionKey("p".into()),
2328                    Bytes::from_static(b"rejected"),
2329                    1,
2330                    (),
2331                    Durability::Buffered
2332                )
2333                .await
2334                .is_err()
2335        );
2336        assert!(engine.flush().await.is_err());
2337    }
2338    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2339    async fn committed_reads_continue_during_new_manifest_publication() {
2340        for fail in [false, true] {
2341            let memory = MemoryBlobStore::new();
2342            let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2343            let seed_sequencer = Arc::new(
2344                ManifestSequencer::open(seed_blob.clone(), "manifest/")
2345                    .await
2346                    .unwrap(),
2347            );
2348            let mut config = FlushConfig::default();
2349            config.max_batches = 1;
2350            config.max_inflight_flushes = 4;
2351            config.linger = Duration::ZERO;
2352            config.budget.enabled = false;
2353            let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2354            seed.produce(
2355                PartitionKey("p".into()),
2356                Bytes::from_static(b"committed"),
2357                1,
2358                (),
2359                Durability::Sequenced,
2360            )
2361            .await
2362            .unwrap();
2363            drop(seed);
2364            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2365            let (release_tx, release_rx) = oneshot::channel();
2366            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2367                inner: memory,
2368                entered: entered_tx,
2369                release: Mutex::new(Some(release_rx)),
2370                fail,
2371                panic_on_commit: false,
2372            });
2373            let sequencer = Arc::new(
2374                ManifestSequencer::open(blob.clone(), "manifest/")
2375                    .await
2376                    .unwrap(),
2377            );
2378            let engine = Arc::new(LogEngine::new(
2379                blob.clone(),
2380                sequencer.clone(),
2381                config,
2382                "data/",
2383            ));
2384            let producing = {
2385                let engine = engine.clone();
2386                tokio::spawn(async move {
2387                    engine
2388                        .produce(
2389                            PartitionKey("p".into()),
2390                            Bytes::from_static(b"new"),
2391                            1,
2392                            (),
2393                            Durability::Sequenced,
2394                        )
2395                        .await
2396                })
2397            };
2398            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2399            let mut reading = tokio::task::spawn_blocking(move || {
2400                let p = PartitionKey("p".into());
2401                (
2402                    sequencer.high_watermark(&p).unwrap(),
2403                    sequencer.lookup(&p, 0).unwrap(),
2404                    sequencer.snapshot(),
2405                )
2406            });
2407            let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2408            let early_ack = producing.is_finished();
2409            // Release even on regression failure so all worker threads can drain.
2410            release_tx.send(()).unwrap();
2411            let outcome = producing.await.unwrap();
2412            assert_eq!(outcome.is_err(), fail);
2413            let reads_progressed = observed.is_ok();
2414            let (watermark, entries, snapshot) = match observed {
2415                Ok(result) => result.unwrap(),
2416                Err(_) => reading.await.unwrap(),
2417            };
2418            assert!(!early_ack);
2419            assert!(
2420                reads_progressed,
2421                "durable manifest publication blocked committed index reads"
2422            );
2423            assert_eq!(watermark, 1);
2424            assert_eq!(entries.len(), 1);
2425            assert_eq!(snapshot.manifest_count, 1);
2426            let rows = engine
2427                .fetch(&PartitionKey("p".into()), 0, 100)
2428                .await
2429                .unwrap();
2430            assert_eq!(rows.len(), if fail { 1 } else { 2 });
2431            assert_eq!(rows[0].payload.as_ref(), b"committed");
2432            drop(engine);
2433            let reopened = Arc::new(
2434                ManifestSequencer::open(blob.clone(), "manifest/")
2435                    .await
2436                    .unwrap(),
2437            );
2438            let engine = LogEngine::new(blob, reopened, config, "data/");
2439            assert_eq!(
2440                engine
2441                    .fetch(&PartitionKey("p".into()), 0, 100)
2442                    .await
2443                    .unwrap()
2444                    .len(),
2445                if fail { 1 } else { 2 }
2446            );
2447        }
2448    }
2449    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2450    async fn manifest_mutations_remain_serialized_while_readers_progress() {
2451        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2452        let (release_tx, release_rx) = oneshot::channel();
2453        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2454            inner: MemoryBlobStore::new(),
2455            entered: entered_tx,
2456            release: Mutex::new(Some(release_rx)),
2457            fail: false,
2458            panic_on_commit: false,
2459        });
2460        let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2461        let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2462            tokio::task::spawn_blocking(move || {
2463                sequencer.commit(&[CommitBatch {
2464                    partition: PartitionKey("p".into()),
2465                    record_count: 1,
2466                    location: BatchLocation {
2467                        object_id: key.into(),
2468                        byte_start: 0,
2469                        byte_len: 1,
2470                    },
2471                    meta: &(),
2472                    epoch: 0,
2473                }])
2474            })
2475        };
2476        let first = commit(sequencer.clone(), "data/first");
2477        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2478        let second = commit(sequencer.clone(), "data/second");
2479        let truncating = {
2480            let sequencer = sequencer.clone();
2481            tokio::task::spawn_blocking(move || {
2482                sequencer.truncate_before(&PartitionKey("p".into()), 1)
2483            })
2484        };
2485        let reading = {
2486            let sequencer = sequencer.clone();
2487            tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2488        };
2489        let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2490        release_tx.send(()).unwrap();
2491        assert_eq!(
2492            observed.expect("committed reads blocked").unwrap().unwrap(),
2493            0
2494        );
2495        let first = tokio::time::timeout(Duration::from_secs(5), first)
2496            .await
2497            .unwrap()
2498            .unwrap()
2499            .unwrap();
2500        let second = tokio::time::timeout(Duration::from_secs(5), second)
2501            .await
2502            .unwrap()
2503            .unwrap()
2504            .unwrap();
2505        let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2506            .await
2507            .unwrap()
2508            .unwrap()
2509            .unwrap();
2510        assert_eq!(
2511            first,
2512            vec![CommitOutcome::Assigned {
2513                base_offset: 0,
2514                record_count: 1
2515            }]
2516        );
2517        assert_eq!(
2518            second,
2519            vec![CommitOutcome::Assigned {
2520                base_offset: 1,
2521                record_count: 1
2522            }]
2523        );
2524        assert_eq!(dropped, vec!["data/first".to_string()]);
2525        let p = PartitionKey("p".into());
2526        assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2527        assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2528        let entries = sequencer.lookup(&p, 0).unwrap();
2529        assert_eq!(entries.len(), 1);
2530        assert_eq!(entries[0].location.object_id, "data/second");
2531        assert_eq!(sequencer.snapshot().manifest_count, 2);
2532    }
2533}
2534
2535#[cfg(test)]
2536mod idle_batch_wait_tests {
2537    use super::*;
2538    use std::sync::mpsc;
2539
2540    fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2541        Arc::new(Shared {
2542            queue: Mutex::new(Queue {
2543                items: VecDeque::new(),
2544                bytes: 0,
2545                bytes_in_use: 0,
2546                shutdown: false,
2547                next_seq: 1,
2548                completed_through: 0,
2549                first_failed: None,
2550                force_flush: false,
2551                flush_waiters: Vec::new(),
2552                last_enqueue: None,
2553                oldest_enqueue: None,
2554            }),
2555            cv: Condvar::new(),
2556            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2557            budget: Mutex::new(BudgetRuntime::new(config.budget)),
2558            flush_config: config,
2559        })
2560    }
2561
2562    #[test]
2563    fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2564        for linger in [Duration::ZERO, Duration::from_millis(1)] {
2565            for shutdown in [false, true] {
2566                let config = FlushConfig {
2567                    linger,
2568                    ..FlushConfig::default()
2569                };
2570                let shared = empty_shared(config);
2571                let (started_tx, started_rx) = mpsc::channel();
2572                let (result_tx, result_rx) = mpsc::channel();
2573                let worker_shared = shared.clone();
2574                let worker = std::thread::spawn(move || {
2575                    started_tx.send(()).unwrap();
2576                    assert!(
2577                        result_tx
2578                            .send(take_batch(&worker_shared, config, true))
2579                            .is_ok()
2580                    );
2581                });
2582                started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2583                let early = result_rx.recv_timeout(Duration::from_millis(100));
2584                let returned_while_idle = early.is_ok();
2585                {
2586                    let mut q = shared.queue.lock().unwrap();
2587                    if shutdown {
2588                        q.shutdown = true;
2589                    } else {
2590                        let payload = Bytes::from_static(b"first-after-idle");
2591                        q.bytes = payload.len();
2592                        q.bytes_in_use = payload.len();
2593                        q.next_seq = 2;
2594                        q.last_enqueue = Some(Instant::now());
2595                        q.oldest_enqueue = q.last_enqueue;
2596                        q.force_flush = true;
2597                        q.items.push_back(Pending {
2598                            partition: PartitionKey("p".into()),
2599                            record_count: 1,
2600                            payload,
2601                            meta: (),
2602                            epoch: 0,
2603                            durability: Durability::Buffered,
2604                            responder: None,
2605                            seq: 1,
2606                        });
2607                    }
2608                    shared.cv.notify_all();
2609                }
2610                let result = early
2611                    .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2612                    .unwrap();
2613                worker.join().unwrap();
2614                assert!(
2615                    !returned_while_idle,
2616                    "idle worker polled with linger={linger:?}"
2617                );
2618                match result {
2619                    TakeBatch::Shutdown => assert!(shutdown),
2620                    TakeBatch::Batch(items) => {
2621                        assert!(!shutdown);
2622                        assert_eq!(items.len(), 1);
2623                        assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2624                        assert_eq!(items[0].seq, 1);
2625                    }
2626                    TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2627                }
2628            }
2629        }
2630    }
2631}