Skip to main content

object_log/
engine.rs

1//! The buffered, multiplexing log engine.
2
3use crate::budget::{
4    BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8    BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21/// The durability point a [`LogEngine::produce`] call resolves at.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24    /// Resolve as soon as the batch is buffered (fire-and-forget; may be lost on
25    /// crash before the flush). No offset is returned.
26    Buffered,
27    /// Resolve once the containing object is durably PUT (survives crash). No
28    /// offset yet — the commit has not run.
29    Durable,
30    /// Resolve once the batch is durably PUT **and** sequenced (has a stable
31    /// offset). This is the strong, no-loss level.
32    Sequenced,
33}
34
35/// Flush-trigger policy for the engine's group-commit buffer.
36#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38    /// Hard ceiling on one sealed object's payload size (memory / object-store
39    /// physics). **Not** the primary packing control: under normal load packing
40    /// is `ingest_rate × linger`. Keep this high enough that **linger binds**
41    /// before size (default **1 GiB**). Lower only for tight RAM or backend caps.
42    pub max_bytes: usize,
43    /// Flush once this many batches are buffered (secondary ceiling).
44    pub max_batches: usize,
45    /// **Maximum** time a batch may wait before a deadline flush (hard ceiling).
46    /// This is the latency↔throughput control surface: longer wait ⇒ more bytes
47    /// per seal ⇒ fewer durable ops/s. The budget controller may use a shorter
48    /// *effective* linger when media is idle (TD-004). `ZERO` = seal as soon as
49    /// any data is buffered (no co-buffer wait). Default `50ms`.
50    pub linger: Duration,
51    /// Max sealed objects PUT concurrently. Default **1** (single-flight bulk
52    /// path). Raise for parallel S3 PUTs. Also bounds ready objects per atomic
53    /// commit when the sequencer opts in to multi-object commits.
54    pub max_inflight_flushes: usize,
55    /// Max bytes in the mutable queue plus in-flight seals. Producers block when
56    /// exceeded. Default **2 GiB**.
57    pub max_buffered_bytes: usize,
58    /// Durable-ops budget controller (default on for Fjord).
59    pub budget: BudgetConfig,
60}
61
62impl Default for FlushConfig {
63    fn default() -> Self {
64        Self {
65            // Physics/safety ceiling so linger defines segment size under load.
66            max_bytes: 1024 * 1024 * 1024,
67            max_batches: 10_000,
68            linger: Duration::from_millis(50),
69            // 1 = single-flight seal (best Local bulk); raise for parallel S3 PUTs.
70            max_inflight_flushes: 1,
71            max_buffered_bytes: 2 * 1024 * 1024 * 1024,
72            budget: BudgetConfig::default(),
73        }
74    }
75}
76
77/// Outcome of a [`LogEngine::produce`] call.
78#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct AppendOutcome {
80    /// First assigned offset, when resolved at [`Durability::Sequenced`].
81    pub base_offset: Option<i64>,
82    /// Last assigned offset, when resolved at [`Durability::Sequenced`].
83    pub last_offset: Option<i64>,
84    /// Whether the batch is durably stored.
85    pub durable: bool,
86    /// Whether the batch has been sequenced (has an offset).
87    pub sequenced: bool,
88}
89
90/// A batch read back by [`LogEngine::fetch`], with its assigned base offset.
91#[derive(Clone, Debug, PartialEq, Eq)]
92pub struct FetchedBatch {
93    /// First offset of the batch.
94    pub base_offset: i64,
95    /// Number of records in the batch.
96    pub record_count: i32,
97    /// The opaque batch payload.
98    pub payload: Bytes,
99}
100
101/// Snapshot of the engine's buffering envelope.
102#[derive(Clone, Debug, PartialEq, Eq)]
103pub struct BufferStats {
104    /// Bytes still waiting in the mutable accumulation queue.
105    pub queued_bytes: usize,
106    /// Bytes owned by queued plus sealed in-flight flush work.
107    pub bytes_in_use: usize,
108    /// Batches still waiting in the mutable accumulation queue.
109    pub queued_batches: usize,
110    /// Configured upper bound for `bytes_in_use`.
111    pub max_buffered_bytes: usize,
112}
113
114type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
115
116fn retryable_storage_error(err: &ObjectLogError) -> bool {
117    matches!(err, ObjectLogError::StorageUnavailable(_))
118}
119
120async fn retry_delay(attempt: usize) {
121    let multiplier = 1u32 << attempt.min(5);
122    tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
123}
124
125async fn put_chunks_with_retries(
126    blob: &Arc<dyn BlobStore>,
127    key: &str,
128    chunks: Vec<Bytes>,
129) -> Result<(), ObjectLogError> {
130    // `Bytes::clone` is refcount-only; Local put_chunks streams without re-merge.
131    let mut attempt = 0usize;
132    loop {
133        match blob.put_chunks(key, chunks.clone()).await {
134            Ok(()) => return Ok(()),
135            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
136                retry_delay(attempt).await;
137                attempt += 1;
138            }
139            Err(err) => return Err(err),
140        }
141    }
142}
143
144async fn get_range_with_retries(
145    blob: &Arc<dyn BlobStore>,
146    key: &str,
147    range: std::ops::Range<u64>,
148) -> Result<Option<Bytes>, ObjectLogError> {
149    let mut attempt = 0usize;
150    loop {
151        match blob.get_range(key, range.clone()).await {
152            Ok(bytes) => return Ok(bytes),
153            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
154                retry_delay(attempt).await;
155                attempt += 1;
156            }
157            Err(err) => return Err(err),
158        }
159    }
160}
161
162struct Pending<M> {
163    partition: PartitionKey,
164    record_count: i32,
165    payload: Bytes,
166    meta: M,
167    /// Fence epoch carried into [`CommitBatch::epoch`].
168    epoch: u64,
169    durability: Durability,
170    responder: Option<Responder>,
171    /// Monotonic enqueue id for [`LogEngine::flush`] barriers.
172    seq: u64,
173}
174
175struct FlushWork<M> {
176    batch: Vec<Pending<M>>,
177    locations: Vec<BatchLocation>,
178    responders: Vec<(Durability, Option<Responder>)>,
179    bytes: usize,
180    /// Highest enqueue seq included in this flush object.
181    max_seq: u64,
182    /// Durable data objects represented by this ordered commit group.
183    data_objects: usize,
184    put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
185    put_started: Instant,
186    put_result: Option<Result<Duration, ObjectLogError>>,
187}
188
189struct CommitJob {
190    task: TokioJoinHandle<usize>,
191    first_seq: u64,
192}
193
194enum TakeBatch<M> {
195    Batch(Vec<Pending<M>>),
196    Empty,
197    Shutdown,
198}
199
200type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
201
202struct Queue<M> {
203    items: VecDeque<Pending<M>>,
204    bytes: usize,
205    bytes_in_use: usize,
206    shutdown: bool,
207    /// Next seq to assign on enqueue.
208    next_seq: u64,
209    /// Highest settled enqueue sequence; failures are retained separately.
210    completed_through: u64,
211    /// Earliest failed enqueue prefix. A later success cannot make it durable.
212    first_failed: Option<(u64, ObjectLogError)>,
213    /// When true, take_batch flushes even if under size/linger.
214    force_flush: bool,
215    /// Waiters: (barrier_seq inclusive, responder).
216    flush_waiters: Vec<FlushWaiter>,
217    /// Last successful produce enqueue (for idle early-flush).
218    last_enqueue: Option<Instant>,
219    /// Enqueue time of the front item (linger deadline = oldest + linger).
220    oldest_enqueue: Option<Instant>,
221}
222
223struct Shared<M> {
224    queue: Mutex<Queue<M>>,
225    cv: Condvar,
226    max_buffered_bytes: usize,
227    /// Budget controller + inspectable counters (TD-004).
228    budget: Mutex<BudgetRuntime>,
229    /// Max linger (operator ceiling) and budget config copy for the flush loop.
230    flush_config: FlushConfig,
231}
232
233/// A buffered, multiplexing append-log engine over a [`BlobStore`], with
234/// sequencing delegated to a [`Sequencer`].
235///
236/// `produce` group-commits: many batches across many partitions are multiplexed
237/// into one object, PUT durably, then handed to the sequencer in a single call —
238/// so PUT count is decoupled from produce count. A single flush worker preserves
239/// per-[`PartitionKey`] arrival order and never splits a partition across
240/// concurrent commits.
241pub struct LogEngine<S: Sequencer> {
242    shared: Arc<Shared<S::Meta>>,
243    blob: Arc<dyn BlobStore>,
244    sequencer: Arc<S>,
245    /// Data-object key prefix (`<prefix><counter:020>`, or
246    /// `<prefix><writer>/<epoch>/<counter>` when a writer id is set).
247    key_prefix: String,
248    writer_id: String,
249    flush_thread: Option<JoinHandle<()>>,
250}
251
252impl<S> LogEngine<S>
253where
254    S: Sequencer + 'static,
255    S::Meta: Send + 'static,
256{
257    /// Create an engine over `blob` and `sequencer` with the given flush policy.
258    /// Objects are keyed `<key_prefix><counter>`; pick a prefix unique to this
259    /// engine instance if several share a store.
260    pub fn new(
261        blob: Arc<dyn BlobStore>,
262        sequencer: Arc<S>,
263        config: FlushConfig,
264        key_prefix: impl Into<String>,
265    ) -> Self {
266        Self::new_with_writer(blob, sequencer, config, key_prefix, "")
267    }
268
269    /// Like [`new`](Self::new), but data objects are
270    /// `<key_prefix><writer_id>/<epoch>/<counter>` so two writers cannot
271    /// overwrite the same key. `writer_id` must not contain `/` or `..`.
272    pub fn new_with_writer(
273        blob: Arc<dyn BlobStore>,
274        sequencer: Arc<S>,
275        config: FlushConfig,
276        key_prefix: impl Into<String>,
277        writer_id: impl Into<String>,
278    ) -> Self {
279        if let Err(msg) = config.budget.validate() {
280            panic!("invalid FlushConfig.budget: {msg}");
281        }
282        if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
283            eprintln!("object-log flush config: {config:?}");
284        }
285        let budget_rt = BudgetRuntime::new(config.budget);
286        let shared = Arc::new(Shared {
287            queue: Mutex::new(Queue {
288                items: VecDeque::new(),
289                bytes: 0,
290                bytes_in_use: 0,
291                shutdown: false,
292                next_seq: 1, // first enqueued batch gets seq 1
293                completed_through: 0,
294                first_failed: None,
295                force_flush: false,
296                flush_waiters: Vec::new(),
297                last_enqueue: None,
298                oldest_enqueue: None,
299            }),
300            cv: Condvar::new(),
301            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
302            budget: Mutex::new(budget_rt),
303            flush_config: config,
304        });
305        let key_prefix = key_prefix.into();
306        let writer_id = writer_id.into();
307        let flush_thread = {
308            let shared = Arc::clone(&shared);
309            let blob = Arc::clone(&blob);
310            let sequencer = Arc::clone(&sequencer);
311            let prefix = key_prefix.clone();
312            let writer_id = writer_id.clone();
313            std::thread::Builder::new()
314                .name("object-log-flush".into())
315                .spawn(move || flush_loop(shared, blob, sequencer, config, prefix, writer_id))
316                .expect("spawn flush thread")
317        };
318        Self {
319            writer_id,
320            shared,
321            blob,
322            sequencer,
323            key_prefix,
324            flush_thread: Some(flush_thread),
325        }
326    }
327
328    /// Prefix used for data objects written by this engine.
329    pub fn data_prefix(&self) -> &str {
330        &self.key_prefix
331    }
332
333    /// Borrow the engine's blob store (e.g. for orphan reaping or inspection).
334    pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
335        &self.blob
336    }
337
338    /// Borrow the engine's sequencer.
339    pub fn sequencer(&self) -> &Arc<S> {
340        &self.sequencer
341    }
342
343    /// Writer id used in data-object keys. Empty when constructed with [`new`](Self::new).
344    pub fn writer_id(&self) -> &str {
345        &self.writer_id
346    }
347
348    /// Buffer a batch and resolve at the requested [`Durability`].
349    ///
350    /// The batch's fence epoch is `0`. Use [`produce_at_epoch`](Self::produce_at_epoch)
351    /// when the partition index is fenced.
352    pub async fn produce(
353        &self,
354        partition: PartitionKey,
355        payload: Bytes,
356        record_count: i32,
357        meta: S::Meta,
358        durability: Durability,
359    ) -> Result<AppendOutcome, ObjectLogError> {
360        self.produce_at_epoch(partition, payload, record_count, meta, durability, 0)
361            .await
362    }
363
364    /// [`produce`](Self::produce) at a caller-supplied fence epoch.
365    pub async fn produce_at_epoch(
366        &self,
367        partition: PartitionKey,
368        payload: Bytes,
369        record_count: i32,
370        meta: S::Meta,
371        durability: Durability,
372        epoch: u64,
373    ) -> Result<AppendOutcome, ObjectLogError> {
374        if record_count <= 0 {
375            return Err(ObjectLogError::InvalidBatch(
376                "record_count must be > 0".into(),
377            ));
378        }
379        if matches!(durability, Durability::Buffered) {
380            self.enqueue(Pending {
381                partition,
382                record_count,
383                payload,
384                meta,
385                epoch,
386                durability,
387                responder: None,
388                seq: 0, // filled in enqueue
389            })?;
390            return Ok(AppendOutcome {
391                base_offset: None,
392                last_offset: None,
393                durable: false,
394                sequenced: false,
395            });
396        }
397        // fail_closed: reserve predicted media ops before waiting on flush.
398        {
399            let mut budget = self.shared.budget.lock().expect("poisoned");
400            if budget.config.enabled
401                && budget.config.mode == BudgetMode::FailClosed
402                && !budget.reserve_for_fail_closed(Instant::now())
403            {
404                return Err(ObjectLogError::BudgetExceeded(
405                    "insufficient durable-ops budget to admit produce".into(),
406                ));
407            }
408        }
409        // budget_priority: wait briefly for tokens when empty.
410        if self.shared.flush_config.budget.enabled
411            && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
412        {
413            let timeout = self.shared.flush_config.budget.admission_timeout;
414            let deadline = Instant::now() + timeout;
415            loop {
416                let mut budget = self.shared.budget.lock().expect("poisoned");
417                if budget.can_admit_now(Instant::now()) {
418                    break;
419                }
420                drop(budget);
421                if Instant::now() >= deadline {
422                    return Err(ObjectLogError::BudgetExceeded(
423                        "timed out waiting for durable-ops budget".into(),
424                    ));
425                }
426                std::thread::sleep(Duration::from_millis(1));
427            }
428        }
429        let (tx, rx) = oneshot::channel();
430        self.enqueue(Pending {
431            partition,
432            record_count,
433            payload,
434            meta,
435            epoch,
436            durability,
437            responder: Some(tx),
438            seq: 0,
439        })?;
440        rx.await
441            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
442    }
443
444    /// Seal every batch enqueued **at or before** this call (barrier).
445    ///
446    /// Use after [`Durability::Buffered`] produces to wait for durable PUT (and
447    /// sequencing) of that prior work. Concurrent produces enqueued after this
448    /// call are not required to finish. An empty successful buffer returns immediately.
449    /// A failed PUT or commit remains an error for every barrier covering that
450    /// enqueue prefix, even after later appends succeed. Reopen the engine to
451    /// establish a new prefix from its successfully committed index.
452    pub async fn flush(&self) -> Result<(), ObjectLogError> {
453        let rx = {
454            let mut q = self.shared.queue.lock().expect("poisoned");
455            if q.shutdown {
456                return Err(ObjectLogError::Sequencer("engine shutting down".into()));
457            }
458            if q.next_seq == 1 {
459                // Nothing has ever been enqueued.
460                return Ok(());
461            }
462            let barrier = q.next_seq - 1;
463            if let Some((sequence, error)) = &q.first_failed {
464                if *sequence <= barrier {
465                    return Err(error.clone());
466                }
467            }
468            if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
469                return Ok(());
470            }
471            let (tx, rx) = oneshot::channel();
472            q.force_flush = true;
473            q.flush_waiters.push((barrier, tx));
474            self.shared.cv.notify_all();
475            rx
476        };
477        rx.await
478            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
479    }
480
481    fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
482        let item_len = item.payload.len();
483        let max_buffered_bytes = self.shared.max_buffered_bytes;
484        let shared = Arc::clone(&self.shared);
485        let mut q = shared.queue.lock().expect("poisoned");
486        while !q.shutdown
487            && q.bytes_in_use > 0
488            && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
489        {
490            q = shared.cv.wait(q).expect("poisoned");
491        }
492        if q.shutdown {
493            return Err(ObjectLogError::Sequencer("engine shutting down".into()));
494        }
495        let now = Instant::now();
496        item.seq = q.next_seq;
497        q.next_seq = q.next_seq.saturating_add(1);
498        q.bytes += item_len;
499        q.bytes_in_use += item_len;
500        q.last_enqueue = Some(now);
501        if q.oldest_enqueue.is_none() {
502            q.oldest_enqueue = Some(now);
503        }
504        q.items.push_back(item);
505        shared.cv.notify_all();
506        Ok(())
507    }
508
509    /// Read batches covering offsets at/after `offset`, up to ~`max_bytes`.
510    pub async fn fetch(
511        &self,
512        partition: &PartitionKey,
513        offset: i64,
514        max_bytes: usize,
515    ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
516        let entries = self.sequencer.lookup(partition, offset)?;
517        let mut out = Vec::new();
518        let mut total = 0usize;
519        for e in entries {
520            if total >= max_bytes && !out.is_empty() {
521                break;
522            }
523            let batch = self.load_entry(&e).await?;
524            total += batch.payload.len();
525            out.push(batch);
526        }
527        Ok(out)
528    }
529
530    /// Stream batches from `offset` onward without materializing a full `Vec`.
531    ///
532    /// Calls `visit` once per index entry in order. Suitable for wide offset
533    /// windows (bounded-RAM replay). Stops and returns the error if `visit`
534    /// fails. Unlike [`Self::fetch`], there is no byte budget — visit every remaining
535    /// batch (or stop yourself inside `visit`).
536    pub async fn fetch_stream<F>(
537        &self,
538        partition: &PartitionKey,
539        offset: i64,
540        mut visit: F,
541    ) -> Result<(), ObjectLogError>
542    where
543        F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
544    {
545        let entries = self.sequencer.lookup(partition, offset)?;
546        for e in entries {
547            visit(self.load_entry(&e).await?)?;
548        }
549        Ok(())
550    }
551
552    async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
553        let start = e.location.byte_start as u64;
554        let end = start + e.location.byte_len as u64;
555        let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
556            .await?
557            .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
558        Ok(FetchedBatch {
559            base_offset: e.base_offset,
560            record_count: e.record_count,
561            payload: bytes,
562        })
563    }
564
565    /// Drop the partition's log below `offset` and delete any object that thereby
566    /// becomes fully unreferenced.
567    pub async fn truncate_before(
568        &self,
569        partition: &PartitionKey,
570        offset: i64,
571    ) -> Result<(), ObjectLogError> {
572        let dead = self.sequencer.truncate_before(partition, offset)?;
573        for object_id in dead {
574            self.blob.delete(&object_id).await?;
575        }
576        Ok(())
577    }
578
579    /// Delete data-prefix objects that are not in `live`.
580    ///
581    /// **Safety:** call only when this engine (and any other writer using the
582    /// same prefix) is quiescent — otherwise an in-flight put that has not yet
583    /// been committed may be deleted as an "orphan". Prefer running offline or
584    /// after drop. Does not touch keys outside [`data_prefix`](Self::data_prefix)
585    /// (e.g. a separate manifest prefix is safe).
586    ///
587    /// Returns the deleted object keys.
588    pub async fn reap_orphans(
589        &self,
590        live: &HashSet<String>,
591    ) -> Result<Vec<String>, ObjectLogError> {
592        reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
593    }
594
595    /// Return a point-in-time snapshot of queued and in-flight payload bytes.
596    pub fn buffer_stats(&self) -> BufferStats {
597        let q = self.shared.queue.lock().expect("poisoned");
598        BufferStats {
599            queued_bytes: q.bytes,
600            bytes_in_use: q.bytes_in_use,
601            queued_batches: q.items.len(),
602            max_buffered_bytes: self.shared.max_buffered_bytes,
603        }
604    }
605
606    /// Inspect budget controller layers and counters (TD-004).
607    pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
608        let now = Instant::now();
609        let mut budget = self.shared.budget.lock().expect("poisoned");
610        budget.refill(now);
611        let effective = budget.effective_budget_per_sec;
612        let reason = if !budget.config.enabled {
613            EffectiveReason::Disabled
614        } else if budget.config.budget_per_sec_cap.is_some()
615            && budget
616                .config
617                .budget_per_sec_cap
618                .is_some_and(|c| (effective - c).abs() < 1e-9)
619        {
620            EffectiveReason::ConfigCap
621        } else if budget.ongoing_capacity.is_some() {
622            EffectiveReason::Ongoing
623        } else if budget.startup_capacity.is_some() {
624            EffectiveReason::StartupProbe
625        } else {
626            EffectiveReason::DefaultCapacity
627        };
628        let (queued, last_enq) = {
629            let q = self.shared.queue.lock().expect("poisoned");
630            (q.bytes, q.last_enqueue)
631        };
632        let early = budget.allow_early_flush(now, queued, last_enq);
633        let max_linger = self.shared.flush_config.linger;
634        let eff_linger = if !budget.config.enabled {
635            max_linger
636        } else if early || max_linger.is_zero() {
637            Duration::ZERO
638        } else {
639            max_linger
640        };
641        PipelineSnapshot {
642            budget_per_sec: EffectiveKnob {
643                configured: budget.config.budget_per_sec_cap,
644                startup_measured: budget.startup_capacity,
645                ongoing_measured: budget.ongoing_capacity,
646                effective,
647                reason,
648            },
649            effective_linger_ms: EffectiveKnob {
650                configured: Some(max_linger.as_millis() as u64),
651                startup_measured: None,
652                ongoing_measured: None,
653                effective: eff_linger.as_millis() as u64,
654                reason: if early {
655                    EffectiveReason::Ongoing
656                } else {
657                    EffectiveReason::Configured
658                },
659            },
660            max_linger_ms: max_linger.as_millis() as u64,
661            token_fill_ratio: budget.fill_ratio(),
662            media_ops_total: budget.media_ops_total,
663            overdraft_total: budget.overdraft_total,
664            flushes_total: budget.flushes_total,
665            undersized_deadline_flushes: budget.undersized_deadline_flushes,
666            predicted_media_ops: budget.predicted_media_ops,
667            budget_enabled: budget.config.enabled,
668            budget_mode: budget.config.mode,
669        }
670    }
671}
672
673/// Delete objects under `data_prefix` whose keys are absent from `live`.
674///
675/// See [`LogEngine::reap_orphans`] for safety notes. `live` is typically
676/// [`InMemorySequencer::live_object_ids`](crate::InMemorySequencer::live_object_ids)
677/// or [`ManifestSequencer::live_object_ids`](crate::ManifestSequencer::live_object_ids).
678pub async fn reap_orphans(
679    blob: &dyn BlobStore,
680    data_prefix: &str,
681    live: &HashSet<String>,
682) -> Result<Vec<String>, ObjectLogError> {
683    let keys = blob.list(data_prefix).await?;
684    let mut deleted = Vec::new();
685    for key in keys {
686        if live.contains(&key) {
687            continue;
688        }
689        blob.delete(&key).await?;
690        deleted.push(key);
691    }
692    Ok(deleted)
693}
694
695impl<S: Sequencer> Drop for LogEngine<S> {
696    fn drop(&mut self) {
697        {
698            let mut q = self.shared.queue.lock().expect("poisoned");
699            q.shutdown = true;
700        }
701        self.shared.cv.notify_all();
702        if let Some(t) = self.flush_thread.take() {
703            let _ = t.join();
704        }
705    }
706}
707
708/// Highest numeric suffix already present under `data_prefix`.
709///
710/// Empty `writer_id` reads legacy keys `{prefix}{counter:020}`. A writer id
711/// reads `{prefix}{writer}/{epoch}/{counter}` and ignores other writers, so a
712/// restarted process with a new writer id cannot overwrite their objects.
713async fn recover_data_object_counter(
714    blob: &dyn BlobStore,
715    prefix: &str,
716    writer_id: &str,
717) -> Result<u64, ObjectLogError> {
718    let keys = blob.list(prefix).await?;
719    let mut max = 0u64;
720    let writer_prefix = format!("{prefix}{writer_id}/");
721    for key in keys {
722        let Some(suffix) = key.strip_prefix(prefix) else {
723            continue;
724        };
725        if writer_id.is_empty() {
726            if let Ok(n) = suffix.parse::<u64>() {
727                max = max.max(n);
728            }
729            continue;
730        }
731        let Some(rest) = key.strip_prefix(&writer_prefix) else {
732            continue;
733        };
734        if let Some(n) = rest
735            .rsplit('/')
736            .next()
737            .and_then(|tail| tail.parse::<u64>().ok())
738        {
739            max = max.max(n);
740        }
741    }
742    Ok(max)
743}
744
745fn data_object_key(prefix: &str, writer_id: &str, epoch: u64, counter: u64) -> String {
746    if writer_id.is_empty() {
747        format!("{prefix}{counter:020}")
748    } else {
749        format!("{prefix}{writer_id}/{epoch:020}/{counter:020}")
750    }
751}
752
753fn flush_loop<S>(
754    shared: Arc<Shared<S::Meta>>,
755    blob: Arc<dyn BlobStore>,
756    sequencer: Arc<S>,
757    config: FlushConfig,
758    prefix: String,
759    writer_id: String,
760) where
761    S: Sequencer + 'static,
762    S::Meta: Send + 'static,
763{
764    let max_inflight = config.max_inflight_flushes.max(1);
765    let commit_group_limit = if sequencer.supports_multi_object_commit() {
766        max_inflight
767    } else {
768        1
769    };
770    let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
771        .ok()
772        .and_then(|v| v.parse::<usize>().ok())
773        .filter(|v| *v > 0)
774        .unwrap_or_else(|| max_inflight.min(8));
775    let rt = tokio::runtime::Builder::new_multi_thread()
776        .worker_threads(worker_threads)
777        .enable_all()
778        .build()
779        .expect("flush runtime");
780    // Resume the data-object counter past any keys already under `prefix`. Restarting
781    // at 0 on reopen overwrites sealed objects while manifests still point at the old
782    // byte ranges → RangeOutOfBounds / mid-JSON EOF on fetch (fireweed-481d3e43).
783    let mut counter = match rt.block_on(recover_data_object_counter(
784        blob.as_ref(),
785        &prefix,
786        &writer_id,
787    )) {
788        Ok(counter) => counter,
789        Err(error) => {
790            // A failed listing is not an empty prefix. Fail admission before
791            // any PUT can reuse a sealed object's name.
792            let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
793            return;
794        }
795    };
796    let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
797    let mut active_puts = 0usize;
798    let mut committing: Option<CommitJob> = None;
799    let mut shutdown = false;
800
801    loop {
802        while !shutdown && active_puts < max_inflight {
803            // Poll the commit job even with no queued uploads: it owns bytes
804            // whose release may be needed to admit the next producer.
805            let wait_for_more = pending.is_empty() && committing.is_none();
806            match take_batch(&shared, config, wait_for_more) {
807                TakeBatch::Batch(batch) => {
808                    counter += 1;
809                    let concurrent = max_inflight > 1;
810                    let work = start_flush_work(
811                        &rt, &blob, &prefix, &writer_id, counter, batch, concurrent,
812                    );
813                    if concurrent {
814                        active_puts += 1;
815                    }
816                    pending.push_back(work);
817                }
818                TakeBatch::Empty => break,
819                TakeBatch::Shutdown => shutdown = true,
820            }
821        }
822
823        let mut made_progress = false;
824        for work in pending.iter_mut() {
825            let Some(put) = work.put.as_ref() else {
826                continue;
827            };
828            if !put.is_finished() {
829                continue;
830            }
831            let put = work.put.take().expect("put handle exists");
832            let elapsed = work.put_started.elapsed();
833            work.put_result = Some(match rt.block_on(put) {
834                Ok(Ok(())) => Ok(elapsed),
835                Ok(Err(e)) => Err(e),
836                Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
837                    "flush task failed: {e}"
838                ))),
839            });
840            active_puts = active_puts.saturating_sub(1);
841            made_progress = true;
842        }
843
844        if committing
845            .as_ref()
846            .is_some_and(|job| job.task.is_finished())
847        {
848            let job = committing.take().expect("completed commit exists");
849            match rt.block_on(job.task) {
850                Ok(released) => {
851                    let mut q = shared.queue.lock().expect("poisoned");
852                    q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
853                    shared.cv.notify_all();
854                }
855                Err(error) => {
856                    let uploads = fail_engine_and_take_pending_uploads(
857                        &shared,
858                        &mut pending,
859                        job.first_seq,
860                        ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
861                    );
862                    // A shared runtime can outlive this engine. Await accepted uploads
863                    // explicitly so close/reopen/reaping cannot race orphan writes.
864                    // They are never sequenced after the failed commit.
865                    for upload in uploads {
866                        let _ = rt.block_on(upload);
867                    }
868                    return;
869                }
870            }
871            made_progress = true;
872        }
873
874        if committing.is_none() {
875            if let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit) {
876                if max_inflight == 1 {
877                    // Preserve the low-overhead single-flight path.
878                    let released = finish_flush_work(&shared, &blob, &sequencer, work);
879                    let mut q = shared.queue.lock().expect("poisoned");
880                    q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
881                    shared.cv.notify_all();
882                } else {
883                    // One ordered committer; uploads can progress during its
884                    // durable manifest I/O. Bytes remain charged until completion.
885                    let first_seq = work
886                        .batch
887                        .iter()
888                        .map(|p| p.seq)
889                        .min()
890                        .unwrap_or(work.max_seq);
891                    let shared = shared.clone();
892                    let blob = blob.clone();
893                    let sequencer = sequencer.clone();
894                    committing = Some(CommitJob {
895                        task: rt.spawn_blocking(move || {
896                            finish_flush_work(&shared, &blob, &sequencer, work)
897                        }),
898                        first_seq,
899                    });
900                }
901                made_progress = true;
902            }
903        }
904
905        if pending.is_empty() && committing.is_none() {
906            {
907                let mut q = shared.queue.lock().expect("poisoned");
908                if q.items.is_empty() && q.bytes_in_use == 0 {
909                    // Nothing in flight: any remaining waiters with barrier already
910                    // covered (or no work after last seal) can complete.
911                    notify_flush_waiters(&mut q);
912                }
913            }
914            shared.cv.notify_all();
915            if shutdown {
916                // Fail any remaining flush waiters.
917                let mut q = shared.queue.lock().expect("poisoned");
918                for (_, tx) in q.flush_waiters.drain(..) {
919                    let _ = tx.send(Err(ObjectLogError::Sequencer(
920                        "engine shutting down".into(),
921                    )));
922                }
923                return;
924            }
925            continue;
926        }
927
928        if !made_progress {
929            std::thread::sleep(Duration::from_millis(1));
930        }
931    }
932}
933
934/// Close admission and fail queued callers/barriers after an unrecoverable
935/// startup or commit failure. Return started uploads for explicit draining.
936fn fail_engine_and_take_pending_uploads<M>(
937    shared: &Arc<Shared<M>>,
938    pending: &mut VecDeque<FlushWork<M>>,
939    first_seq: u64,
940    error: ObjectLogError,
941) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
942    let mut uploads = Vec::new();
943    for mut work in pending.drain(..) {
944        if let Some(put) = work.put.take() {
945            uploads.push(put);
946        }
947        send_storage_error(&mut work.responders, error.clone());
948    }
949    // Recover a poisoned guard only to close the engine, never to resume writes.
950    let mut q = shared
951        .queue
952        .lock()
953        .unwrap_or_else(|poisoned| poisoned.into_inner());
954    q.shutdown = true;
955    q.first_failed.get_or_insert((first_seq, error.clone()));
956    for mut item in q.items.drain(..) {
957        if let Some(tx) = item.responder.take() {
958            let _ = tx.send(Err(error.clone()));
959        }
960    }
961    q.bytes = 0;
962    // Keep outstanding bytes conservatively charged until the failed engine
963    // and its accepted uploads have completed.
964    for (_, tx) in q.flush_waiters.drain(..) {
965        let _ = tx.send(Err(error.clone()));
966    }
967    q.force_flush = false;
968    shared.cv.notify_all();
969    uploads
970}
971
972/// Combine only an already-ready contiguous success prefix. Never wait to fill
973/// a group or cross an unfinished/failed PUT; preserve object and batch order.
974fn take_ready_commit_group<M>(
975    pending: &mut VecDeque<FlushWork<M>>,
976    max_objects: usize,
977) -> Option<FlushWork<M>> {
978    pending.front()?.put_result.as_ref()?;
979    let mut group = pending.pop_front().expect("ready front exists");
980    if group.put_result.as_ref().is_some_and(Result::is_err) {
981        return Some(group);
982    }
983    while group.data_objects < max_objects.max(1) {
984        let Some(next) = pending.front() else { break };
985        if !next.put_result.as_ref().is_some_and(Result::is_ok)
986            || group.data_objects.saturating_add(next.data_objects) > max_objects
987        {
988            break;
989        }
990        let mut next = pending.pop_front().expect("ready successor exists");
991        group.batch.append(&mut next.batch);
992        group.locations.append(&mut next.locations);
993        group.responders.append(&mut next.responders);
994        group.bytes = group
995            .bytes
996            .checked_add(next.bytes)
997            .expect("bounded pending bytes");
998        group.max_seq = group.max_seq.max(next.max_seq);
999        group.data_objects += next.data_objects;
1000        let next_elapsed = next.put_result.take().unwrap().unwrap();
1001        let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
1002        *elapsed = (*elapsed).max(next_elapsed);
1003    }
1004    Some(group)
1005}
1006
1007/// How long to sleep before re-evaluating a seal decision.
1008/// - `ZERO` means seal now (early-flush or linger deadline hit).
1009/// - Otherwise wait at most this long (interrupted by new produces).
1010fn effective_linger<M>(
1011    shared: &Shared<M>,
1012    config: FlushConfig,
1013    queued_bytes: usize,
1014    last_enqueue: Option<Instant>,
1015    oldest_enqueue: Option<Instant>,
1016) -> Duration {
1017    if config.linger.is_zero() {
1018        return Duration::ZERO;
1019    }
1020    let now = Instant::now();
1021    if config.budget.enabled {
1022        let mut budget = shared.budget.lock().expect("poisoned");
1023        budget.refill(now);
1024        if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
1025            budget.note_early_flush(now);
1026            return Duration::ZERO;
1027        }
1028    }
1029    // True linger: seal when the oldest buffered item reaches max wait.
1030    if let Some(oldest) = oldest_enqueue {
1031        let deadline = oldest + config.linger;
1032        if now >= deadline {
1033            return Duration::ZERO;
1034        }
1035        let until_deadline = deadline.saturating_duration_since(now);
1036        // Also wake when idle gate could open, so sparse produces don't wait full linger.
1037        if config.budget.enabled
1038            && !config.budget.early_flush_idle.is_zero()
1039            && let Some(last) = last_enqueue
1040        {
1041            let idle_at = last + config.budget.early_flush_idle;
1042            if idle_at > now && idle_at < deadline {
1043                return idle_at.saturating_duration_since(now);
1044            }
1045        }
1046        return until_deadline;
1047    }
1048    config.linger
1049}
1050
1051fn take_batch<M>(
1052    shared: &Arc<Shared<M>>,
1053    config: FlushConfig,
1054    wait_when_empty: bool,
1055) -> TakeBatch<M> {
1056    let mut q = shared.queue.lock().expect("poisoned");
1057    loop {
1058        if q.items.is_empty() {
1059            if q.shutdown {
1060                return TakeBatch::Shutdown;
1061            }
1062            if !wait_when_empty {
1063                return TakeBatch::Empty;
1064            }
1065            // No pending work needs a timer. Enqueue and shutdown change the
1066            // predicate under this mutex and notify. Waiting without
1067            // dropping/reacquiring it first also avoids a lost notification.
1068            q = shared.cv.wait(q).expect("poisoned");
1069            continue;
1070        }
1071
1072        let linger = {
1073            let queued = q.bytes;
1074            let last_enq = q.last_enqueue;
1075            let oldest = q.oldest_enqueue;
1076            drop(q);
1077            let l = effective_linger(shared, config, queued, last_enq, oldest);
1078            q = shared.queue.lock().expect("poisoned");
1079            l
1080        };
1081
1082        let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1083        let force = q.force_flush;
1084        let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1085        if triggered {
1086            if !size_trigger && !q.shutdown && !q.items.is_empty() {
1087                let mut budget = shared.budget.lock().expect("poisoned");
1088                if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1089                    budget.undersized_deadline_flushes += 1;
1090                }
1091            }
1092            break;
1093        }
1094
1095        // Short wait (until idle gate or linger deadline); re-evaluate, do not
1096        // seal solely because a probe sleep timed out.
1097        let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1098        q = guard;
1099    }
1100
1101    let force_drain = q.force_flush;
1102    let mut items = Vec::new();
1103    let mut bytes = 0usize;
1104    // On force_flush, still respect max_bytes so one object stays bounded, but
1105    // ignore max_batches so a barrier can drain large queues across objects.
1106    while let Some(item) = q.items.pop_front() {
1107        q.bytes = q.bytes.saturating_sub(item.payload.len());
1108        bytes += item.payload.len();
1109        items.push(item);
1110        if bytes >= config.max_bytes {
1111            break;
1112        }
1113        if !force_drain && items.len() >= config.max_batches {
1114            break;
1115        }
1116    }
1117    q.oldest_enqueue = if q.items.is_empty() {
1118        None
1119    } else {
1120        // Approximate: next seal window starts now for remaining items.
1121        Some(Instant::now())
1122    };
1123    // Keep forcing while a barrier still has queued work.
1124    if force_drain {
1125        q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1126    }
1127    if items.is_empty() {
1128        TakeBatch::Empty
1129    } else {
1130        TakeBatch::Batch(items)
1131    }
1132}
1133
1134fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1135    let done = q.completed_through;
1136    let mut i = 0;
1137    while i < q.flush_waiters.len() {
1138        let barrier = q.flush_waiters[i].0;
1139        let failure = q
1140            .first_failed
1141            .as_ref()
1142            .filter(|(sequence, _)| *sequence <= barrier)
1143            .map(|(_, error)| error.clone());
1144        if barrier <= done || failure.is_some() {
1145            let (_, tx) = q.flush_waiters.swap_remove(i);
1146            let _ = tx.send(failure.map_or(Ok(()), Err));
1147        } else {
1148            i += 1;
1149        }
1150    }
1151    if q.flush_waiters.is_empty() {
1152        q.force_flush = false;
1153    } else if q.items.is_empty() && q.bytes_in_use == 0 {
1154        // Barriers still waiting but no work — should not happen; clear force.
1155        q.force_flush = true;
1156    } else {
1157        q.force_flush = true;
1158    }
1159}
1160
1161fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1162    for (_, tx) in responders.iter_mut() {
1163        if let Some(tx) = tx.take() {
1164            let _ = tx.send(Err(err.clone()));
1165        }
1166    }
1167}
1168
1169fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1170    for (durability, tx) in responders.iter_mut() {
1171        if *durability == Durability::Durable
1172            && let Some(tx) = tx.take()
1173        {
1174            let _ = tx.send(Ok(AppendOutcome {
1175                base_offset: None,
1176                last_offset: None,
1177                durable: true,
1178                sequenced: false,
1179            }));
1180        }
1181    }
1182}
1183
1184fn prepare_flush_work<M>(
1185    prefix: &str,
1186    writer_id: &str,
1187    counter: u64,
1188    mut batch: Vec<Pending<M>>,
1189) -> (FlushWork<M>, String, Vec<Bytes>) {
1190    let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1191    let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1192    let epoch = batch.first().map(|pending| pending.epoch).unwrap_or(0);
1193    let key = data_object_key(prefix, writer_id, epoch, counter);
1194    let mut offset = 0usize;
1195    for p in &batch {
1196        let start = offset as u32;
1197        offset += p.payload.len();
1198        chunks.push(p.payload.clone());
1199        locations.push(BatchLocation {
1200            object_id: key.clone(),
1201            byte_start: start,
1202            byte_len: p.payload.len() as u32,
1203        });
1204    }
1205    if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1206        eprintln!(
1207            "object-log seal: key={key} batches={} bytes={offset}",
1208            batch.len()
1209        );
1210    }
1211    let responders: Vec<(Durability, Option<Responder>)> = batch
1212        .iter_mut()
1213        .map(|p| (p.durability, p.responder.take()))
1214        .collect();
1215    let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1216    let work = FlushWork {
1217        batch,
1218        locations,
1219        responders,
1220        bytes: offset,
1221        max_seq,
1222        data_objects: 1,
1223        put: None,
1224        put_started: Instant::now(),
1225        put_result: None,
1226    };
1227    (work, key, chunks)
1228}
1229
1230/// Start a durable put for a sealed batch.
1231///
1232/// - `concurrent == false` (default `max_inflight_flushes == 1`): `block_on` the
1233///   put on the flush thread; Local uses `block_in_place` (no spawn_blocking queue).
1234/// - `concurrent == true`: spawn put tasks for parallel S3-style throughput.
1235fn start_flush_work<M>(
1236    rt: &tokio::runtime::Runtime,
1237    blob: &Arc<dyn BlobStore>,
1238    prefix: &str,
1239    writer_id: &str,
1240    counter: u64,
1241    batch: Vec<Pending<M>>,
1242    concurrent: bool,
1243) -> FlushWork<M> {
1244    let (mut work, key, chunks) = prepare_flush_work(prefix, writer_id, counter, batch);
1245    let blob = Arc::clone(blob);
1246    let _ = blob.take_media_op_stats();
1247    work.put_started = Instant::now();
1248    if concurrent {
1249        work.put =
1250            Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
1251    } else {
1252        let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1253        let elapsed = work.put_started.elapsed();
1254        work.put_result = Some(match put_result {
1255            Ok(()) => Ok(elapsed),
1256            Err(e) => Err(e),
1257        });
1258    }
1259    work
1260}
1261
1262fn finish_flush_work<S, M>(
1263    shared: &Arc<Shared<M>>,
1264    blob: &Arc<dyn BlobStore>,
1265    sequencer: &Arc<S>,
1266    mut work: FlushWork<S::Meta>,
1267) -> usize
1268where
1269    S: Sequencer + 'static,
1270    S::Meta: Send + 'static,
1271{
1272    let release_bytes = work.bytes;
1273    let first_seq = work
1274        .batch
1275        .iter()
1276        .map(|pending| pending.seq)
1277        .min()
1278        .unwrap_or(work.max_seq);
1279    // Durable-then-sequence: the object PUT may have overlapped later PUTs, but
1280    // sequencer commits are still completed in object creation order.
1281    // Grouped put_ms is the largest observed member PUT duration, not a sum.
1282    let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1283    let put_elapsed = match work.put_result.take().expect("put result is ready") {
1284        Ok(elapsed) => elapsed,
1285        Err(e) => {
1286            send_storage_error(&mut work.responders, e.clone());
1287            let mut q = shared.queue.lock().expect("poisoned");
1288            // Unblock flush waiters covering this seq with the storage error.
1289            let done = work.max_seq;
1290            if done >= q.completed_through {
1291                q.completed_through = done;
1292            }
1293            q.first_failed.get_or_insert((first_seq, e));
1294            notify_flush_waiters(&mut q);
1295            shared.cv.notify_all();
1296            return release_bytes;
1297        }
1298    };
1299
1300    // Media ops for the data object put.
1301    let mut media_ops = blob
1302        .take_media_op_stats()
1303        .map(|s| s.media_ops)
1304        .unwrap_or(work.data_objects as u64); // fallback: 1 per successful put
1305
1306    // Signal Durable-level waiters now (after PUT, before commit).
1307    send_durable_acks(&mut work.responders);
1308
1309    // Sequence the ready group atomically, preserving each object's locations.
1310    let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1311        .batch
1312        .iter()
1313        .zip(work.locations.iter())
1314        .map(|(p, loc)| CommitBatch {
1315            partition: p.partition.clone(),
1316            record_count: p.record_count,
1317            location: loc.clone(),
1318            meta: &p.meta,
1319            epoch: p.epoch,
1320        })
1321        .collect();
1322
1323    let commit_started = timing.then(Instant::now);
1324    // Clear stats again so sequencer durable work (e.g. ManifestSequencer put)
1325    // on the shared store is counted into the same budget.
1326    let _ = blob.take_media_op_stats();
1327    let mut commit_error = None;
1328    match sequencer.commit(&commit_batches) {
1329        Ok(outcomes) => {
1330            if let Some(commit_started) = commit_started {
1331                eprintln!(
1332                    "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1333                    work.bytes,
1334                    commit_batches.len(),
1335                    put_elapsed.as_millis(),
1336                    commit_started.elapsed().as_millis(),
1337                    work.data_objects
1338                );
1339            }
1340            if let Some(s) = blob.take_media_op_stats() {
1341                media_ops = media_ops.saturating_add(s.media_ops);
1342            }
1343            for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1344                if let Some(tx) = tx.take() {
1345                    let (base, last) = match outcome {
1346                        CommitOutcome::Assigned {
1347                            base_offset,
1348                            record_count,
1349                        } => (
1350                            Some(base_offset),
1351                            Some(base_offset + record_count as i64 - 1),
1352                        ),
1353                        CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1354                        CommitOutcome::Rejected { reason } => {
1355                            let _ = tx.send(Err(ObjectLogError::Sequencer(reason)));
1356                            continue;
1357                        }
1358                    };
1359                    let _ = tx.send(Ok(AppendOutcome {
1360                        base_offset: base,
1361                        last_offset: last,
1362                        durable: true,
1363                        sequenced: true,
1364                    }));
1365                }
1366            }
1367        }
1368        Err(e) => {
1369            if let Some(s) = blob.take_media_op_stats() {
1370                media_ops = media_ops.saturating_add(s.media_ops);
1371            }
1372            send_storage_error(&mut work.responders, e.clone());
1373            commit_error = Some(e);
1374        }
1375    }
1376
1377    {
1378        let mut budget = shared.budget.lock().expect("poisoned");
1379        budget.consume_after_flush(media_ops, Instant::now());
1380    }
1381    {
1382        let mut q = shared.queue.lock().expect("poisoned");
1383        if let Some(error) = commit_error {
1384            q.first_failed.get_or_insert((first_seq, error));
1385        }
1386        if work.max_seq >= q.completed_through {
1387            q.completed_through = work.max_seq;
1388        }
1389        notify_flush_waiters(&mut q);
1390    }
1391    shared.cv.notify_all();
1392
1393    release_bytes
1394}
1395
1396#[cfg(test)]
1397mod flush_barrier_tests {
1398    use super::*;
1399
1400    #[test]
1401    fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1402        let error = ObjectLogError::Sequencer("manifest failed".into());
1403        let mut queue = Queue::<()> {
1404            items: VecDeque::new(),
1405            bytes: 0,
1406            bytes_in_use: 1,
1407            shutdown: false,
1408            next_seq: 4,
1409            completed_through: 2,
1410            first_failed: Some((2, error.clone())),
1411            force_flush: true,
1412            flush_waiters: Vec::new(),
1413            last_enqueue: None,
1414            oldest_enqueue: None,
1415        };
1416        let mut receivers = Vec::new();
1417        for barrier in [1, 2, 3] {
1418            let (tx, rx) = oneshot::channel();
1419            queue.flush_waiters.push((barrier, tx));
1420            receivers.push(rx);
1421        }
1422        notify_flush_waiters(&mut queue);
1423        assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1424        assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1425        assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1426        assert!(queue.flush_waiters.is_empty());
1427    }
1428}
1429
1430#[cfg(test)]
1431mod ready_commit_group_tests {
1432    use super::*;
1433    use crate::{ManifestSequencer, MemoryBlobStore};
1434    use async_trait::async_trait;
1435    use std::ops::Range;
1436
1437    fn work(
1438        n: u64,
1439        durability: Durability,
1440    ) -> (
1441        FlushWork<()>,
1442        String,
1443        Vec<Bytes>,
1444        oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1445    ) {
1446        let (tx, rx) = oneshot::channel();
1447        let (mut work, key, chunks) = prepare_flush_work(
1448            "data/",
1449            "",
1450            n,
1451            vec![Pending {
1452                partition: PartitionKey("shared".into()),
1453                record_count: 1,
1454                payload: Bytes::from(format!("body-{n}")),
1455                meta: (),
1456                epoch: 0,
1457                durability,
1458                responder: Some(tx),
1459                seq: n,
1460            }],
1461        );
1462        work.put_result = Some(Ok(Duration::from_millis(n)));
1463        (work, key, chunks, rx)
1464    }
1465
1466    #[test]
1467    fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1468        let mut pending = (1..=5)
1469            .map(|n| work(n, Durability::Sequenced).0)
1470            .collect::<VecDeque<_>>();
1471        let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1472        let mut released = 0;
1473        for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1474            let group = take_ready_commit_group(&mut pending, 2).unwrap();
1475            assert_eq!(
1476                group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1477                expected
1478            );
1479            assert_eq!(group.data_objects, expected.len());
1480            assert_eq!(group.responders.len(), expected.len());
1481            assert_eq!(group.max_seq, *expected.last().unwrap());
1482            for (location, n) in group.locations.iter().zip(&expected) {
1483                assert_eq!(location.object_id, format!("data/{n:020}"));
1484                assert_eq!(location.byte_start, 0);
1485                assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1486            }
1487            released += group.bytes;
1488        }
1489        assert!(pending.is_empty());
1490        assert_eq!(released, original_bytes);
1491
1492        let mut first = work(1, Durability::Sequenced).0;
1493        first.put_result = None;
1494        let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1495        assert!(take_ready_commit_group(&mut pending, 8).is_none());
1496        assert_eq!(
1497            pending.len(),
1498            2,
1499            "a later ready PUT cannot pass an unfinished head"
1500        );
1501        pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1502        let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1503        assert_eq!(failed.data_objects, 1);
1504        assert!(failed.put_result.unwrap().is_err());
1505        assert_eq!(pending[0].max_seq, 2);
1506
1507        let mut failed = work(2, Durability::Sequenced).0;
1508        failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1509        let mut pending = VecDeque::from([
1510            work(1, Durability::Sequenced).0,
1511            failed,
1512            work(3, Durability::Sequenced).0,
1513        ]);
1514        assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1515        assert_eq!(
1516            pending.len(),
1517            2,
1518            "a success group cannot cross a failed PUT"
1519        );
1520        let mut pending = VecDeque::from([
1521            work(1, Durability::Sequenced).0,
1522            work(2, Durability::Sequenced).0,
1523        ]);
1524        assert_eq!(
1525            take_ready_commit_group(&mut pending, 1)
1526                .unwrap()
1527                .data_objects,
1528            1
1529        );
1530        assert_eq!(
1531            pending.len(),
1532            1,
1533            "one-object sequencers retain their call boundary"
1534        );
1535    }
1536
1537    struct GatedManifestStore {
1538        inner: MemoryBlobStore,
1539        entered: std::sync::mpsc::SyncSender<()>,
1540        release: Mutex<Option<oneshot::Receiver<()>>>,
1541        fail: bool,
1542        panic_on_commit: bool,
1543    }
1544
1545    #[async_trait]
1546    impl BlobStore for GatedManifestStore {
1547        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1548            if key.starts_with("manifest/") {
1549                let release = self.release.lock().unwrap().take();
1550                if let Some(release) = release {
1551                    self.entered.send(()).unwrap();
1552                    release.await.unwrap();
1553                    assert!(!self.panic_on_commit, "injected commit panic");
1554                    if self.fail {
1555                        return Err(ObjectLogError::StorageUnavailable(
1556                            "manifest failure".into(),
1557                        ));
1558                    }
1559                }
1560            }
1561            self.inner.put(key, value).await
1562        }
1563        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1564            self.inner.get(key).await
1565        }
1566        async fn get_range(
1567            &self,
1568            key: &str,
1569            range: Range<u64>,
1570        ) -> Result<Option<Bytes>, ObjectLogError> {
1571            self.inner.get_range(key, range).await
1572        }
1573        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1574            self.inner.list(prefix).await
1575        }
1576        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1577            self.inner.delete(key).await
1578        }
1579        fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1580            self.inner.take_media_op_stats()
1581        }
1582    }
1583
1584    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1585    async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1586        for fail in [false, true] {
1587            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1588            let (release_tx, release_rx) = oneshot::channel();
1589            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1590                inner: MemoryBlobStore::new(),
1591                entered: entered_tx,
1592                release: Mutex::new(Some(release_rx)),
1593                fail,
1594                panic_on_commit: false,
1595            });
1596            let sequencer = Arc::new(
1597                ManifestSequencer::open(blob.clone(), "manifest/")
1598                    .await
1599                    .unwrap(),
1600            );
1601            assert!(sequencer.supports_multi_object_commit());
1602            let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1603            let (mut second, second_key, second_chunks, mut durable_rx) =
1604                work(2, Durability::Durable);
1605            second.batch[0].partition = PartitionKey("other".into());
1606            let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1607            blob.put_chunks(&first_key, first_chunks).await.unwrap();
1608            blob.put_chunks(&second_key, second_chunks).await.unwrap();
1609            blob.put_chunks(&third_key, third_chunks).await.unwrap();
1610            let mut pending = VecDeque::from([first, second, third]);
1611            let group = take_ready_commit_group(&mut pending, 8).unwrap();
1612            let bytes = group.bytes;
1613            assert_eq!(group.data_objects, 3);
1614            let (flush_tx, mut flush_rx) = oneshot::channel();
1615            let mut config = FlushConfig::default();
1616            config.budget.enabled = false;
1617            let shared: Arc<Shared<()>> = Arc::new(Shared {
1618                queue: Mutex::new(Queue {
1619                    items: VecDeque::new(),
1620                    bytes: 0,
1621                    bytes_in_use: bytes,
1622                    shutdown: false,
1623                    next_seq: 4,
1624                    completed_through: 0,
1625                    first_failed: None,
1626                    force_flush: true,
1627                    flush_waiters: vec![(3, flush_tx)],
1628                    last_enqueue: None,
1629                    oldest_enqueue: None,
1630                }),
1631                cv: Condvar::new(),
1632                max_buffered_bytes: bytes,
1633                budget: Mutex::new(BudgetRuntime::new(config.budget)),
1634                flush_config: config,
1635            });
1636            let worker = {
1637                let shared = shared.clone();
1638                let blob = blob.clone();
1639                let sequencer = sequencer.clone();
1640                std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1641            };
1642            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1643            assert!(matches!(
1644                sequenced_rx.try_recv(),
1645                Err(oneshot::error::TryRecvError::Empty)
1646            ));
1647            assert!(matches!(
1648                third_rx.try_recv(),
1649                Err(oneshot::error::TryRecvError::Empty)
1650            ));
1651            assert!(matches!(
1652                flush_rx.try_recv(),
1653                Err(oneshot::error::TryRecvError::Empty)
1654            ));
1655            let durable = durable_rx.try_recv().unwrap().unwrap();
1656            assert!(durable.durable && !durable.sequenced);
1657            release_tx.send(()).unwrap();
1658            assert_eq!(worker.join().unwrap(), bytes);
1659            if fail {
1660                assert!(sequenced_rx.await.unwrap().is_err());
1661                assert!(third_rx.await.unwrap().is_err());
1662                assert!(flush_rx.await.unwrap().is_err());
1663                assert_eq!(
1664                    sequencer
1665                        .high_watermark(&PartitionKey("shared".into()))
1666                        .unwrap(),
1667                    0
1668                );
1669                assert!(blob.list("manifest/").await.unwrap().is_empty());
1670            } else {
1671                assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1672                assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1673                flush_rx.await.unwrap().unwrap();
1674                assert_eq!(blob.list("manifest/").await.unwrap().len(), 3);
1675            }
1676            assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1677            let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1678                .await
1679                .unwrap();
1680            for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1681                let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1682                assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1683                for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1684                    assert_eq!(entry.base_offset, offset as i64);
1685                    let loc = &entry.location;
1686                    assert_eq!(loc.object_id, format!("data/{object:020}"));
1687                    let body = blob
1688                        .get_range(
1689                            &loc.object_id,
1690                            loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1691                        )
1692                        .await
1693                        .unwrap()
1694                        .unwrap();
1695                    assert_eq!(body, format!("body-{object}"));
1696                }
1697            }
1698        }
1699    }
1700
1701    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1702    async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1703        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1704        let (release_tx, release_rx) = oneshot::channel();
1705        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1706            inner: MemoryBlobStore::new(),
1707            entered: entered_tx,
1708            release: Mutex::new(Some(release_rx)),
1709            fail: false,
1710            panic_on_commit: false,
1711        });
1712        let sequencer = Arc::new(
1713            ManifestSequencer::open(blob.clone(), "manifest/")
1714                .await
1715                .unwrap(),
1716        );
1717        let mut config = FlushConfig::default();
1718        config.max_batches = 1;
1719        config.max_inflight_flushes = 4;
1720        config.linger = Duration::ZERO;
1721        config.budget.enabled = false;
1722        let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1723        let first = {
1724            let engine = engine.clone();
1725            tokio::spawn(async move {
1726                engine
1727                    .produce(
1728                        PartitionKey("p".into()),
1729                        Bytes::from_static(b"first"),
1730                        1,
1731                        (),
1732                        Durability::Sequenced,
1733                    )
1734                    .await
1735            })
1736        };
1737        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1738        for n in 1..4 {
1739            engine
1740                .produce(
1741                    PartitionKey("p".into()),
1742                    Bytes::from(format!("later-{n}")),
1743                    1,
1744                    (),
1745                    Durability::Buffered,
1746                )
1747                .await
1748                .unwrap();
1749        }
1750        let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1751            loop {
1752                if blob.list("data/").await.unwrap().len() == 4 {
1753                    break;
1754                }
1755                tokio::time::sleep(Duration::from_millis(1)).await;
1756            }
1757        })
1758        .await
1759        .is_ok();
1760        let early_ack = first.is_finished();
1761        // Always release before assertions so a failed regression can drain.
1762        release_tx.send(()).unwrap();
1763        first.await.unwrap().unwrap();
1764        engine.flush().await.unwrap();
1765        let rows = engine
1766            .fetch(&PartitionKey("p".into()), 0, 1024)
1767            .await
1768            .unwrap();
1769        assert_eq!(
1770            rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
1771            vec![0, 1, 2, 3]
1772        );
1773        drop(engine);
1774        assert!(
1775            !early_ack,
1776            "sequenced acknowledgement preceded manifest durability"
1777        );
1778        assert!(
1779            upload_progress,
1780            "blocked manifest prevented independent data uploads"
1781        );
1782    }
1783
1784    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1785    async fn commit_completion_releases_admission_bytes_without_more_uploads() {
1786        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1787        let sequencer = Arc::new(
1788            ManifestSequencer::open(blob.clone(), "manifest/")
1789                .await
1790                .unwrap(),
1791        );
1792        let mut config = FlushConfig::default();
1793        config.max_bytes = 4;
1794        config.max_buffered_bytes = 4;
1795        config.max_batches = 1;
1796        config.max_inflight_flushes = 4;
1797        config.linger = Duration::ZERO;
1798        config.budget.enabled = false;
1799        let engine = LogEngine::new(blob, sequencer, config, "data/");
1800        engine
1801            .produce(
1802                PartitionKey("p".into()),
1803                Bytes::from_static(b"full"),
1804                1,
1805                (),
1806                Durability::Sequenced,
1807            )
1808            .await
1809            .unwrap();
1810        tokio::time::timeout(Duration::from_secs(5), async {
1811            while engine.buffer_stats().bytes_in_use != 0 {
1812                tokio::time::sleep(Duration::from_millis(1)).await;
1813            }
1814        })
1815        .await
1816        .expect("completed committer stranded admission bytes");
1817        engine
1818            .produce(
1819                PartitionKey("p".into()),
1820                Bytes::from_static(b"next"),
1821                1,
1822                (),
1823                Durability::Sequenced,
1824            )
1825            .await
1826            .unwrap();
1827        engine.flush().await.unwrap();
1828        assert_eq!(
1829            engine
1830                .fetch(&PartitionKey("p".into()), 0, 100)
1831                .await
1832                .unwrap()
1833                .len(),
1834            2
1835        );
1836    }
1837
1838    struct FailureInjectingStore {
1839        inner: Arc<dyn BlobStore>,
1840        second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
1841        second_upload_completed: std::sync::atomic::AtomicBool,
1842        fail_data_listing: std::sync::atomic::AtomicBool,
1843    }
1844
1845    impl FailureInjectingStore {
1846        fn new(inner: Arc<dyn BlobStore>) -> Self {
1847            Self {
1848                inner,
1849                second_upload_gate: Mutex::new(None),
1850                second_upload_completed: std::sync::atomic::AtomicBool::new(false),
1851                fail_data_listing: std::sync::atomic::AtomicBool::new(false),
1852            }
1853        }
1854    }
1855
1856    #[async_trait]
1857    impl BlobStore for FailureInjectingStore {
1858        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1859            let gate = if key == "data/00000000000000000002" {
1860                self.second_upload_gate.lock().unwrap().take()
1861            } else {
1862                None
1863            };
1864            if let Some((entered, release)) = gate {
1865                entered.send(()).unwrap();
1866                release.await.unwrap();
1867                let result = self.inner.put(key, value).await;
1868                if result.is_ok() {
1869                    self.second_upload_completed
1870                        .store(true, std::sync::atomic::Ordering::SeqCst);
1871                }
1872                return result;
1873            }
1874            self.inner.put(key, value).await
1875        }
1876        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1877            self.inner.get(key).await
1878        }
1879        async fn get_range(
1880            &self,
1881            key: &str,
1882            range: Range<u64>,
1883        ) -> Result<Option<Bytes>, ObjectLogError> {
1884            self.inner.get_range(key, range).await
1885        }
1886        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1887            if prefix == "data/"
1888                && self
1889                    .fail_data_listing
1890                    .load(std::sync::atomic::Ordering::SeqCst)
1891            {
1892                return Err(ObjectLogError::StorageUnavailable(
1893                    "injected object listing failure".into(),
1894                ));
1895            }
1896            self.inner.list(prefix).await
1897        }
1898        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1899            self.inner.delete(key).await
1900        }
1901    }
1902
1903    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1904    async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
1905        let inner = Arc::new(MemoryBlobStore::new());
1906        let key = "data/00000000000000000001";
1907        let original = Bytes::from_static(b"existing immutable object");
1908        inner.put(key, original.clone()).await.unwrap();
1909        let store = Arc::new(FailureInjectingStore::new(inner.clone()));
1910        store
1911            .fail_data_listing
1912            .store(true, std::sync::atomic::Ordering::SeqCst);
1913        let mut config = FlushConfig::default();
1914        config.max_batches = 1;
1915        config.max_inflight_flushes = 4;
1916        config.linger = Duration::ZERO;
1917        config.budget.enabled = false;
1918        let engine = LogEngine::new(
1919            store,
1920            Arc::new(crate::InMemorySequencer::new()),
1921            config,
1922            "data/",
1923        );
1924        let result = tokio::time::timeout(
1925            Duration::from_secs(5),
1926            engine.produce(
1927                PartitionKey("p".into()),
1928                Bytes::from_static(b"must not overwrite"),
1929                1,
1930                (),
1931                Durability::Sequenced,
1932            ),
1933        )
1934        .await;
1935        let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
1936        tokio::task::spawn_blocking(move || drop(engine))
1937            .await
1938            .unwrap();
1939        let retained = inner.get(key).await.unwrap();
1940        assert_eq!(
1941            retained,
1942            Some(original),
1943            "failed listing must preserve sealed bytes"
1944        );
1945        assert!(
1946            result.unwrap().is_err(),
1947            "failed prefix recovery must reject admission"
1948        );
1949        assert!(
1950            flushed.unwrap().is_err(),
1951            "flush must report failed initialization"
1952        );
1953        assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
1954    }
1955
1956    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1957    async fn engines_commit_and_close_independently() {
1958        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1959        let (release_tx, release_rx) = oneshot::channel();
1960        let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1961            inner: MemoryBlobStore::new(),
1962            entered: entered_tx,
1963            release: Mutex::new(Some(release_rx)),
1964            fail: false,
1965            panic_on_commit: false,
1966        })));
1967        let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
1968        let mut config = FlushConfig::default();
1969        config.max_batches = 1;
1970        config.max_inflight_flushes = 4;
1971        config.linger = Duration::ZERO;
1972        config.budget.enabled = false;
1973        let a_seq = Arc::new(
1974            ManifestSequencer::open(a_blob.clone(), "manifest/")
1975                .await
1976                .unwrap(),
1977        );
1978        let b_seq = Arc::new(
1979            ManifestSequencer::open(b_blob.clone(), "manifest/")
1980                .await
1981                .unwrap(),
1982        );
1983        let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
1984        a.produce(
1985            PartitionKey("p".into()),
1986            Bytes::from_static(b"a-first"),
1987            1,
1988            (),
1989            Durability::Buffered,
1990        )
1991        .await
1992        .unwrap();
1993        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1994        let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
1995        let b_result = tokio::time::timeout(
1996            Duration::from_secs(5),
1997            b.produce(
1998                PartitionKey("p".into()),
1999                Bytes::from_static(b"b-first"),
2000                1,
2001                (),
2002                Durability::Sequenced,
2003            ),
2004        )
2005        .await;
2006        let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
2007        let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
2008        // Always release the held manifest before assertions, including the pre-change red run.
2009        release_tx.send(()).unwrap();
2010        if b_closed.is_err() {
2011            closing_b.await.unwrap();
2012        }
2013        assert!(
2014            b_closed.is_ok(),
2015            "closing B waited for A's blocked manifest"
2016        );
2017        assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
2018        a.flush().await.unwrap();
2019        assert_eq!(
2020            a.produce(
2021                PartitionKey("p".into()),
2022                Bytes::from_static(b"a-second"),
2023                1,
2024                (),
2025                Durability::Sequenced
2026            )
2027            .await
2028            .unwrap()
2029            .base_offset,
2030            Some(1)
2031        );
2032        let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
2033        assert_eq!(
2034            rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
2035            vec![
2036                Bytes::from_static(b"a-first"),
2037                Bytes::from_static(b"a-second")
2038            ]
2039        );
2040        tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
2041    }
2042
2043    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2044    async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
2045        let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
2046        let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
2047        let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
2048        let (upload_release_tx, upload_release_rx) = oneshot::channel();
2049        let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2050            inner: MemoryBlobStore::new(),
2051            entered: manifest_entered_tx,
2052            release: Mutex::new(Some(manifest_release_rx)),
2053            fail: false,
2054            panic_on_commit: true,
2055        })));
2056        *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
2057        let mut config = FlushConfig::default();
2058        config.max_batches = 1;
2059        config.max_inflight_flushes = 4;
2060        config.linger = Duration::ZERO;
2061        config.budget.enabled = false;
2062        let sequencer = Arc::new(
2063            ManifestSequencer::open(store.clone(), "manifest/")
2064                .await
2065                .unwrap(),
2066        );
2067        let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
2068        engine
2069            .produce(
2070                PartitionKey("p".into()),
2071                Bytes::from_static(b"first"),
2072                1,
2073                (),
2074                Durability::Buffered,
2075            )
2076            .await
2077            .unwrap();
2078        manifest_entered_rx
2079            .recv_timeout(Duration::from_secs(5))
2080            .unwrap();
2081        engine
2082            .produce(
2083                PartitionKey("p".into()),
2084                Bytes::from_static(b"second"),
2085                1,
2086                (),
2087                Durability::Buffered,
2088            )
2089            .await
2090            .unwrap();
2091        upload_entered_rx
2092            .recv_timeout(Duration::from_secs(5))
2093            .unwrap();
2094        manifest_release_tx.send(()).unwrap();
2095        tokio::time::timeout(Duration::from_secs(5), async {
2096            while !engine.shared.queue.lock().unwrap().shutdown {
2097                tokio::time::sleep(Duration::from_millis(1)).await;
2098            }
2099        })
2100        .await
2101        .unwrap();
2102        let flush_failed = engine.flush().await.is_err();
2103        // A failed committer closes admission first; its accepted PUT must still
2104        // complete before close returns, even though no manifest may reference it.
2105        let upload_was_owned = upload_release_tx.send(()).is_ok();
2106        tokio::time::timeout(
2107            Duration::from_secs(5),
2108            tokio::task::spawn_blocking(move || drop(engine)),
2109        )
2110        .await
2111        .unwrap()
2112        .unwrap();
2113        assert!(flush_failed);
2114        assert!(upload_was_owned);
2115        assert!(
2116            store
2117                .second_upload_completed
2118                .load(std::sync::atomic::Ordering::SeqCst)
2119        );
2120        assert_eq!(store.list("data/").await.unwrap().len(), 2);
2121        let sequencer = Arc::new(
2122            ManifestSequencer::open(store.clone(), "manifest/")
2123                .await
2124                .unwrap(),
2125        );
2126        let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2127        assert!(
2128            reopened
2129                .fetch(&PartitionKey("p".into()), 0, 100)
2130                .await
2131                .unwrap()
2132                .is_empty()
2133        );
2134        assert_eq!(
2135            reopened
2136                .produce(
2137                    PartitionKey("p".into()),
2138                    Bytes::from_static(b"after-reopen"),
2139                    1,
2140                    (),
2141                    Durability::Sequenced
2142                )
2143                .await
2144                .unwrap()
2145                .base_offset,
2146            Some(0)
2147        );
2148        let rows = reopened
2149            .fetch(&PartitionKey("p".into()), 0, 100)
2150            .await
2151            .unwrap();
2152        assert_eq!(rows.len(), 1);
2153        assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2154        assert_eq!(store.list("data/").await.unwrap().len(), 3);
2155        tokio::task::spawn_blocking(move || drop(reopened))
2156            .await
2157            .unwrap();
2158    }
2159
2160    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2161    async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2162        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2163        let (release_tx, release_rx) = oneshot::channel();
2164        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2165            inner: MemoryBlobStore::new(),
2166            entered: entered_tx,
2167            release: Mutex::new(Some(release_rx)),
2168            fail: false,
2169            panic_on_commit: false,
2170        });
2171        let sequencer = Arc::new(
2172            ManifestSequencer::open(blob.clone(), "manifest/")
2173                .await
2174                .unwrap(),
2175        );
2176        let mut config = FlushConfig::default();
2177        config.max_batches = 1;
2178        config.max_inflight_flushes = 4;
2179        config.linger = Duration::ZERO;
2180        config.budget.enabled = false;
2181        let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2182        engine
2183            .produce(
2184                PartitionKey("p".into()),
2185                Bytes::from_static(b"first"),
2186                1,
2187                (),
2188                Durability::Buffered,
2189            )
2190            .await
2191            .unwrap();
2192        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2193        engine
2194            .produce(
2195                PartitionKey("p".into()),
2196                Bytes::from_static(b"second"),
2197                1,
2198                (),
2199                Durability::Buffered,
2200            )
2201            .await
2202            .unwrap();
2203        let shared = engine.shared.clone();
2204        let dropping = tokio::task::spawn_blocking(move || drop(engine));
2205        tokio::time::timeout(Duration::from_secs(5), async {
2206            while !shared.queue.lock().unwrap().shutdown {
2207                tokio::time::sleep(Duration::from_millis(1)).await;
2208            }
2209        })
2210        .await
2211        .unwrap();
2212        let premature = dropping.is_finished();
2213        release_tx.send(()).unwrap();
2214        tokio::time::timeout(Duration::from_secs(5), dropping)
2215            .await
2216            .unwrap()
2217            .unwrap();
2218        assert!(!premature, "shutdown returned before manifest durability");
2219        let reopened = Arc::new(
2220            ManifestSequencer::open(blob.clone(), "manifest/")
2221                .await
2222                .unwrap(),
2223        );
2224        let engine = LogEngine::new(blob, reopened, config, "data/");
2225        let rows = engine
2226            .fetch(&PartitionKey("p".into()), 0, 100)
2227            .await
2228            .unwrap();
2229        assert_eq!(
2230            rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2231            vec![b"first".as_slice(), b"second".as_slice()]
2232        );
2233    }
2234
2235    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2236    async fn panicked_committer_closes_admission_and_fails_flush() {
2237        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2238        let (release_tx, release_rx) = oneshot::channel();
2239        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2240            inner: MemoryBlobStore::new(),
2241            entered: entered_tx,
2242            release: Mutex::new(Some(release_rx)),
2243            fail: false,
2244            panic_on_commit: true,
2245        });
2246        let sequencer = Arc::new(
2247            ManifestSequencer::open(blob.clone(), "manifest/")
2248                .await
2249                .unwrap(),
2250        );
2251        let mut config = FlushConfig::default();
2252        config.max_batches = 1;
2253        config.max_inflight_flushes = 4;
2254        config.linger = Duration::ZERO;
2255        config.budget.enabled = false;
2256        let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2257        let first = {
2258            let engine = engine.clone();
2259            tokio::spawn(async move {
2260                engine
2261                    .produce(
2262                        PartitionKey("p".into()),
2263                        Bytes::from_static(b"first"),
2264                        1,
2265                        (),
2266                        Durability::Sequenced,
2267                    )
2268                    .await
2269            })
2270        };
2271        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2272        engine
2273            .produce(
2274                PartitionKey("p".into()),
2275                Bytes::from_static(b"later"),
2276                1,
2277                (),
2278                Durability::Buffered,
2279            )
2280            .await
2281            .unwrap();
2282        let flushing = {
2283            let engine = engine.clone();
2284            tokio::spawn(async move { engine.flush().await })
2285        };
2286        release_tx.send(()).unwrap();
2287        assert!(
2288            tokio::time::timeout(Duration::from_secs(5), first)
2289                .await
2290                .unwrap()
2291                .unwrap()
2292                .is_err()
2293        );
2294        assert!(
2295            tokio::time::timeout(Duration::from_secs(5), flushing)
2296                .await
2297                .unwrap()
2298                .unwrap()
2299                .is_err()
2300        );
2301        tokio::time::timeout(Duration::from_secs(5), async {
2302            while !engine.shared.queue.lock().unwrap().shutdown {
2303                tokio::time::sleep(Duration::from_millis(1)).await;
2304            }
2305        })
2306        .await
2307        .unwrap();
2308        assert!(
2309            engine
2310                .produce(
2311                    PartitionKey("p".into()),
2312                    Bytes::from_static(b"rejected"),
2313                    1,
2314                    (),
2315                    Durability::Buffered
2316                )
2317                .await
2318                .is_err()
2319        );
2320        assert!(engine.flush().await.is_err());
2321    }
2322    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2323    async fn committed_reads_continue_during_new_manifest_publication() {
2324        for fail in [false, true] {
2325            let memory = MemoryBlobStore::new();
2326            let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2327            let seed_sequencer = Arc::new(
2328                ManifestSequencer::open(seed_blob.clone(), "manifest/")
2329                    .await
2330                    .unwrap(),
2331            );
2332            let mut config = FlushConfig::default();
2333            config.max_batches = 1;
2334            config.max_inflight_flushes = 4;
2335            config.linger = Duration::ZERO;
2336            config.budget.enabled = false;
2337            let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2338            seed.produce(
2339                PartitionKey("p".into()),
2340                Bytes::from_static(b"committed"),
2341                1,
2342                (),
2343                Durability::Sequenced,
2344            )
2345            .await
2346            .unwrap();
2347            drop(seed);
2348            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2349            let (release_tx, release_rx) = oneshot::channel();
2350            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2351                inner: memory,
2352                entered: entered_tx,
2353                release: Mutex::new(Some(release_rx)),
2354                fail,
2355                panic_on_commit: false,
2356            });
2357            let sequencer = Arc::new(
2358                ManifestSequencer::open(blob.clone(), "manifest/")
2359                    .await
2360                    .unwrap(),
2361            );
2362            let engine = Arc::new(LogEngine::new(
2363                blob.clone(),
2364                sequencer.clone(),
2365                config,
2366                "data/",
2367            ));
2368            let producing = {
2369                let engine = engine.clone();
2370                tokio::spawn(async move {
2371                    engine
2372                        .produce(
2373                            PartitionKey("p".into()),
2374                            Bytes::from_static(b"new"),
2375                            1,
2376                            (),
2377                            Durability::Sequenced,
2378                        )
2379                        .await
2380                })
2381            };
2382            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2383            let mut reading = tokio::task::spawn_blocking(move || {
2384                let p = PartitionKey("p".into());
2385                (
2386                    sequencer.high_watermark(&p).unwrap(),
2387                    sequencer.lookup(&p, 0).unwrap(),
2388                    sequencer.snapshot(),
2389                )
2390            });
2391            let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2392            let early_ack = producing.is_finished();
2393            // Release even on regression failure so all worker threads can drain.
2394            release_tx.send(()).unwrap();
2395            let outcome = producing.await.unwrap();
2396            assert_eq!(outcome.is_err(), fail);
2397            let reads_progressed = observed.is_ok();
2398            let (watermark, entries, snapshot) = match observed {
2399                Ok(result) => result.unwrap(),
2400                Err(_) => reading.await.unwrap(),
2401            };
2402            assert!(!early_ack);
2403            assert!(
2404                reads_progressed,
2405                "durable manifest publication blocked committed index reads"
2406            );
2407            assert_eq!(watermark, 1);
2408            assert_eq!(entries.len(), 1);
2409            assert_eq!(snapshot.manifest_count, 1);
2410            let rows = engine
2411                .fetch(&PartitionKey("p".into()), 0, 100)
2412                .await
2413                .unwrap();
2414            assert_eq!(rows.len(), if fail { 1 } else { 2 });
2415            assert_eq!(rows[0].payload.as_ref(), b"committed");
2416            drop(engine);
2417            let reopened = Arc::new(
2418                ManifestSequencer::open(blob.clone(), "manifest/")
2419                    .await
2420                    .unwrap(),
2421            );
2422            let engine = LogEngine::new(blob, reopened, config, "data/");
2423            assert_eq!(
2424                engine
2425                    .fetch(&PartitionKey("p".into()), 0, 100)
2426                    .await
2427                    .unwrap()
2428                    .len(),
2429                if fail { 1 } else { 2 }
2430            );
2431        }
2432    }
2433    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2434    async fn manifest_mutations_remain_serialized_while_readers_progress() {
2435        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2436        let (release_tx, release_rx) = oneshot::channel();
2437        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2438            inner: MemoryBlobStore::new(),
2439            entered: entered_tx,
2440            release: Mutex::new(Some(release_rx)),
2441            fail: false,
2442            panic_on_commit: false,
2443        });
2444        let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2445        let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2446            tokio::task::spawn_blocking(move || {
2447                sequencer.commit(&[CommitBatch {
2448                    partition: PartitionKey("p".into()),
2449                    record_count: 1,
2450                    location: BatchLocation {
2451                        object_id: key.into(),
2452                        byte_start: 0,
2453                        byte_len: 1,
2454                    },
2455                    meta: &(),
2456                    epoch: 0,
2457                }])
2458            })
2459        };
2460        let first = commit(sequencer.clone(), "data/first");
2461        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2462        let second = commit(sequencer.clone(), "data/second");
2463        let truncating = {
2464            let sequencer = sequencer.clone();
2465            tokio::task::spawn_blocking(move || {
2466                sequencer.truncate_before(&PartitionKey("p".into()), 1)
2467            })
2468        };
2469        let reading = {
2470            let sequencer = sequencer.clone();
2471            tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2472        };
2473        let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2474        release_tx.send(()).unwrap();
2475        assert_eq!(
2476            observed.expect("committed reads blocked").unwrap().unwrap(),
2477            0
2478        );
2479        let first = tokio::time::timeout(Duration::from_secs(5), first)
2480            .await
2481            .unwrap()
2482            .unwrap()
2483            .unwrap();
2484        let second = tokio::time::timeout(Duration::from_secs(5), second)
2485            .await
2486            .unwrap()
2487            .unwrap()
2488            .unwrap();
2489        let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2490            .await
2491            .unwrap()
2492            .unwrap()
2493            .unwrap();
2494        assert_eq!(
2495            first,
2496            vec![CommitOutcome::Assigned {
2497                base_offset: 0,
2498                record_count: 1
2499            }]
2500        );
2501        assert_eq!(
2502            second,
2503            vec![CommitOutcome::Assigned {
2504                base_offset: 1,
2505                record_count: 1
2506            }]
2507        );
2508        assert_eq!(dropped, vec!["data/first".to_string()]);
2509        let p = PartitionKey("p".into());
2510        assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2511        assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2512        let entries = sequencer.lookup(&p, 0).unwrap();
2513        assert_eq!(entries.len(), 1);
2514        assert_eq!(entries[0].location.object_id, "data/second");
2515        assert_eq!(sequencer.snapshot().manifest_count, 2);
2516    }
2517}
2518
2519#[cfg(test)]
2520mod idle_batch_wait_tests {
2521    use super::*;
2522    use std::sync::mpsc;
2523
2524    fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2525        Arc::new(Shared {
2526            queue: Mutex::new(Queue {
2527                items: VecDeque::new(),
2528                bytes: 0,
2529                bytes_in_use: 0,
2530                shutdown: false,
2531                next_seq: 1,
2532                completed_through: 0,
2533                first_failed: None,
2534                force_flush: false,
2535                flush_waiters: Vec::new(),
2536                last_enqueue: None,
2537                oldest_enqueue: None,
2538            }),
2539            cv: Condvar::new(),
2540            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2541            budget: Mutex::new(BudgetRuntime::new(config.budget)),
2542            flush_config: config,
2543        })
2544    }
2545
2546    #[test]
2547    fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2548        for linger in [Duration::ZERO, Duration::from_millis(1)] {
2549            for shutdown in [false, true] {
2550                let config = FlushConfig {
2551                    linger,
2552                    ..FlushConfig::default()
2553                };
2554                let shared = empty_shared(config);
2555                let (started_tx, started_rx) = mpsc::channel();
2556                let (result_tx, result_rx) = mpsc::channel();
2557                let worker_shared = shared.clone();
2558                let worker = std::thread::spawn(move || {
2559                    started_tx.send(()).unwrap();
2560                    assert!(
2561                        result_tx
2562                            .send(take_batch(&worker_shared, config, true))
2563                            .is_ok()
2564                    );
2565                });
2566                started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2567                let early = result_rx.recv_timeout(Duration::from_millis(100));
2568                let returned_while_idle = early.is_ok();
2569                {
2570                    let mut q = shared.queue.lock().unwrap();
2571                    if shutdown {
2572                        q.shutdown = true;
2573                    } else {
2574                        let payload = Bytes::from_static(b"first-after-idle");
2575                        q.bytes = payload.len();
2576                        q.bytes_in_use = payload.len();
2577                        q.next_seq = 2;
2578                        q.last_enqueue = Some(Instant::now());
2579                        q.oldest_enqueue = q.last_enqueue;
2580                        q.force_flush = true;
2581                        q.items.push_back(Pending {
2582                            partition: PartitionKey("p".into()),
2583                            record_count: 1,
2584                            payload,
2585                            meta: (),
2586                            epoch: 0,
2587                            durability: Durability::Buffered,
2588                            responder: None,
2589                            seq: 1,
2590                        });
2591                    }
2592                    shared.cv.notify_all();
2593                }
2594                let result = early
2595                    .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2596                    .unwrap();
2597                worker.join().unwrap();
2598                assert!(
2599                    !returned_while_idle,
2600                    "idle worker polled with linger={linger:?}"
2601                );
2602                match result {
2603                    TakeBatch::Shutdown => assert!(shutdown),
2604                    TakeBatch::Batch(items) => {
2605                        assert!(!shutdown);
2606                        assert_eq!(items.len(), 1);
2607                        assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2608                        assert_eq!(items[0].seq, 1);
2609                    }
2610                    TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2611                }
2612            }
2613        }
2614    }
2615}