Skip to main content

object_log/
engine.rs

1//! The buffered, multiplexing log engine.
2
3use crate::budget::{
4    BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8    BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21/// The durability point a [`LogEngine::produce`] call resolves at.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24    /// Resolve as soon as the batch is buffered (fire-and-forget; may be lost on
25    /// crash before the flush). No offset is returned.
26    Buffered,
27    /// Resolve once the containing object is durably PUT (survives crash). No
28    /// offset yet — the commit has not run.
29    Durable,
30    /// Resolve once the batch is durably PUT **and** sequenced (has a stable
31    /// offset). This is the strong, no-loss level.
32    Sequenced,
33}
34
35/// Flush-trigger policy for the engine's group-commit buffer.
36#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38    /// Hard ceiling on one sealed object's payload size (memory / object-store
39    /// physics). **Not** the primary packing control: under normal load packing
40    /// is `ingest_rate × linger`. Keep this high enough that **linger binds**
41    /// before size (default **1 GiB**). Lower only for tight RAM or backend caps.
42    pub max_bytes: usize,
43    /// Flush once this many batches are buffered (secondary ceiling).
44    pub max_batches: usize,
45    /// **Maximum** time a batch may wait before a deadline flush (hard ceiling).
46    /// This is the latency↔throughput control surface: longer wait ⇒ more bytes
47    /// per seal ⇒ fewer durable ops/s. The budget controller may use a shorter
48    /// *effective* linger when media is idle (TD-004). `ZERO` = seal as soon as
49    /// any data is buffered (no co-buffer wait). Default `50ms`.
50    pub linger: Duration,
51    /// Max sealed objects PUT concurrently. Default **1** (single-flight bulk
52    /// path). Raise for parallel S3 PUTs. Also bounds ready objects per atomic
53    /// commit when the sequencer opts in to multi-object commits.
54    pub max_inflight_flushes: usize,
55    /// Max bytes in the mutable queue plus in-flight seals. Producers block when
56    /// exceeded. Default **2 GiB**.
57    pub max_buffered_bytes: usize,
58    /// Durable-ops budget controller (default on for Fjord).
59    pub budget: BudgetConfig,
60}
61
62impl Default for FlushConfig {
63    fn default() -> Self {
64        Self {
65            // Physics/safety ceiling so linger defines segment size under load.
66            max_bytes: 1024 * 1024 * 1024,
67            max_batches: 10_000,
68            linger: Duration::from_millis(50),
69            // 1 = single-flight seal (best Local bulk); raise for parallel S3 PUTs.
70            max_inflight_flushes: 1,
71            max_buffered_bytes: 2 * 1024 * 1024 * 1024,
72            budget: BudgetConfig::default(),
73        }
74    }
75}
76
77/// Outcome of a [`LogEngine::produce`] call.
78#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct AppendOutcome {
80    /// First assigned offset, when resolved at [`Durability::Sequenced`].
81    pub base_offset: Option<i64>,
82    /// Last assigned offset, when resolved at [`Durability::Sequenced`].
83    pub last_offset: Option<i64>,
84    /// Whether the batch is durably stored.
85    pub durable: bool,
86    /// Whether the batch has been sequenced (has an offset).
87    pub sequenced: bool,
88}
89
90/// A batch read back by [`LogEngine::fetch`], with its assigned base offset.
91#[derive(Clone, Debug, PartialEq, Eq)]
92pub struct FetchedBatch {
93    /// First offset of the batch.
94    pub base_offset: i64,
95    /// Number of records in the batch.
96    pub record_count: i32,
97    /// The opaque batch payload.
98    pub payload: Bytes,
99}
100
101/// Snapshot of the engine's buffering envelope.
102#[derive(Clone, Debug, PartialEq, Eq)]
103pub struct BufferStats {
104    /// Bytes still waiting in the mutable accumulation queue.
105    pub queued_bytes: usize,
106    /// Bytes owned by queued plus sealed in-flight flush work.
107    pub bytes_in_use: usize,
108    /// Batches still waiting in the mutable accumulation queue.
109    pub queued_batches: usize,
110    /// Configured upper bound for `bytes_in_use`.
111    pub max_buffered_bytes: usize,
112}
113
114type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
115
116fn retryable_storage_error(err: &ObjectLogError) -> bool {
117    matches!(err, ObjectLogError::StorageUnavailable(_))
118}
119
120async fn retry_delay(attempt: usize) {
121    let multiplier = 1u32 << attempt.min(5);
122    tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
123}
124
125async fn put_chunks_with_retries(
126    blob: &Arc<dyn BlobStore>,
127    key: &str,
128    chunks: Vec<Bytes>,
129) -> Result<(), ObjectLogError> {
130    // `Bytes::clone` is refcount-only; Local put_chunks streams without re-merge.
131    let mut attempt = 0usize;
132    loop {
133        match blob.put_chunks(key, chunks.clone()).await {
134            Ok(()) => return Ok(()),
135            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
136                retry_delay(attempt).await;
137                attempt += 1;
138            }
139            Err(err) => return Err(err),
140        }
141    }
142}
143
144async fn get_range_with_retries(
145    blob: &Arc<dyn BlobStore>,
146    key: &str,
147    range: std::ops::Range<u64>,
148) -> Result<Option<Bytes>, ObjectLogError> {
149    let mut attempt = 0usize;
150    loop {
151        match blob.get_range(key, range.clone()).await {
152            Ok(bytes) => return Ok(bytes),
153            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
154                retry_delay(attempt).await;
155                attempt += 1;
156            }
157            Err(err) => return Err(err),
158        }
159    }
160}
161
162struct Pending<M> {
163    partition: PartitionKey,
164    record_count: i32,
165    payload: Bytes,
166    meta: M,
167    durability: Durability,
168    responder: Option<Responder>,
169    /// Monotonic enqueue id for [`LogEngine::flush`] barriers.
170    seq: u64,
171}
172
173struct FlushWork<M> {
174    batch: Vec<Pending<M>>,
175    locations: Vec<BatchLocation>,
176    responders: Vec<(Durability, Option<Responder>)>,
177    bytes: usize,
178    /// Highest enqueue seq included in this flush object.
179    max_seq: u64,
180    /// Durable data objects represented by this ordered commit group.
181    data_objects: usize,
182    put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
183    put_started: Instant,
184    put_result: Option<Result<Duration, ObjectLogError>>,
185}
186
187struct CommitJob {
188    task: TokioJoinHandle<usize>,
189    first_seq: u64,
190}
191
192enum TakeBatch<M> {
193    Batch(Vec<Pending<M>>),
194    Empty,
195    Shutdown,
196}
197
198type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
199
200struct Queue<M> {
201    items: VecDeque<Pending<M>>,
202    bytes: usize,
203    bytes_in_use: usize,
204    shutdown: bool,
205    /// Next seq to assign on enqueue.
206    next_seq: u64,
207    /// Highest settled enqueue sequence; failures are retained separately.
208    completed_through: u64,
209    /// Earliest failed enqueue prefix. A later success cannot make it durable.
210    first_failed: Option<(u64, ObjectLogError)>,
211    /// When true, take_batch flushes even if under size/linger.
212    force_flush: bool,
213    /// Waiters: (barrier_seq inclusive, responder).
214    flush_waiters: Vec<FlushWaiter>,
215    /// Last successful produce enqueue (for idle early-flush).
216    last_enqueue: Option<Instant>,
217    /// Enqueue time of the front item (linger deadline = oldest + linger).
218    oldest_enqueue: Option<Instant>,
219}
220
221struct Shared<M> {
222    queue: Mutex<Queue<M>>,
223    cv: Condvar,
224    max_buffered_bytes: usize,
225    /// Budget controller + inspectable counters (TD-004).
226    budget: Mutex<BudgetRuntime>,
227    /// Max linger (operator ceiling) and budget config copy for the flush loop.
228    flush_config: FlushConfig,
229}
230
231/// A buffered, multiplexing append-log engine over a [`BlobStore`], with
232/// sequencing delegated to a [`Sequencer`].
233///
234/// `produce` group-commits: many batches across many partitions are multiplexed
235/// into one object, PUT durably, then handed to the sequencer in a single call —
236/// so PUT count is decoupled from produce count. A single flush worker preserves
237/// per-[`PartitionKey`] arrival order and never splits a partition across
238/// concurrent commits.
239pub struct LogEngine<S: Sequencer> {
240    shared: Arc<Shared<S::Meta>>,
241    blob: Arc<dyn BlobStore>,
242    sequencer: Arc<S>,
243    /// Data-object key prefix (`<prefix><counter:020>`). Used by orphan reaping.
244    key_prefix: String,
245    flush_thread: Option<JoinHandle<()>>,
246}
247
248impl<S> LogEngine<S>
249where
250    S: Sequencer + 'static,
251    S::Meta: Send + 'static,
252{
253    /// Create an engine over `blob` and `sequencer` with the given flush policy.
254    /// Objects are keyed `<key_prefix><counter>`; pick a prefix unique to this
255    /// engine instance if several share a store.
256    pub fn new(
257        blob: Arc<dyn BlobStore>,
258        sequencer: Arc<S>,
259        config: FlushConfig,
260        key_prefix: impl Into<String>,
261    ) -> Self {
262        if let Err(msg) = config.budget.validate() {
263            panic!("invalid FlushConfig.budget: {msg}");
264        }
265        if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
266            eprintln!("object-log flush config: {config:?}");
267        }
268        let budget_rt = BudgetRuntime::new(config.budget);
269        let shared = Arc::new(Shared {
270            queue: Mutex::new(Queue {
271                items: VecDeque::new(),
272                bytes: 0,
273                bytes_in_use: 0,
274                shutdown: false,
275                next_seq: 1, // first enqueued batch gets seq 1
276                completed_through: 0,
277                first_failed: None,
278                force_flush: false,
279                flush_waiters: Vec::new(),
280                last_enqueue: None,
281                oldest_enqueue: None,
282            }),
283            cv: Condvar::new(),
284            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
285            budget: Mutex::new(budget_rt),
286            flush_config: config,
287        });
288        let key_prefix = key_prefix.into();
289        let flush_thread = {
290            let shared = Arc::clone(&shared);
291            let blob = Arc::clone(&blob);
292            let sequencer = Arc::clone(&sequencer);
293            let prefix = key_prefix.clone();
294            std::thread::Builder::new()
295                .name("object-log-flush".into())
296                .spawn(move || flush_loop(shared, blob, sequencer, config, prefix))
297                .expect("spawn flush thread")
298        };
299        Self {
300            shared,
301            blob,
302            sequencer,
303            key_prefix,
304            flush_thread: Some(flush_thread),
305        }
306    }
307
308    /// Prefix used for data objects written by this engine.
309    pub fn data_prefix(&self) -> &str {
310        &self.key_prefix
311    }
312
313    /// Borrow the engine's blob store (e.g. for orphan reaping or inspection).
314    pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
315        &self.blob
316    }
317
318    /// Borrow the engine's sequencer.
319    pub fn sequencer(&self) -> &Arc<S> {
320        &self.sequencer
321    }
322
323    /// Buffer a batch and resolve at the requested [`Durability`].
324    pub async fn produce(
325        &self,
326        partition: PartitionKey,
327        payload: Bytes,
328        record_count: i32,
329        meta: S::Meta,
330        durability: Durability,
331    ) -> Result<AppendOutcome, ObjectLogError> {
332        if record_count <= 0 {
333            return Err(ObjectLogError::InvalidBatch(
334                "record_count must be > 0".into(),
335            ));
336        }
337        if matches!(durability, Durability::Buffered) {
338            self.enqueue(Pending {
339                partition,
340                record_count,
341                payload,
342                meta,
343                durability,
344                responder: None,
345                seq: 0, // filled in enqueue
346            })?;
347            return Ok(AppendOutcome {
348                base_offset: None,
349                last_offset: None,
350                durable: false,
351                sequenced: false,
352            });
353        }
354        // fail_closed: reserve predicted media ops before waiting on flush.
355        {
356            let mut budget = self.shared.budget.lock().expect("poisoned");
357            if budget.config.enabled
358                && budget.config.mode == BudgetMode::FailClosed
359                && !budget.reserve_for_fail_closed(Instant::now())
360            {
361                return Err(ObjectLogError::BudgetExceeded(
362                    "insufficient durable-ops budget to admit produce".into(),
363                ));
364            }
365        }
366        // budget_priority: wait briefly for tokens when empty.
367        if self.shared.flush_config.budget.enabled
368            && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
369        {
370            let timeout = self.shared.flush_config.budget.admission_timeout;
371            let deadline = Instant::now() + timeout;
372            loop {
373                let mut budget = self.shared.budget.lock().expect("poisoned");
374                if budget.can_admit_now(Instant::now()) {
375                    break;
376                }
377                drop(budget);
378                if Instant::now() >= deadline {
379                    return Err(ObjectLogError::BudgetExceeded(
380                        "timed out waiting for durable-ops budget".into(),
381                    ));
382                }
383                std::thread::sleep(Duration::from_millis(1));
384            }
385        }
386        let (tx, rx) = oneshot::channel();
387        self.enqueue(Pending {
388            partition,
389            record_count,
390            payload,
391            meta,
392            durability,
393            responder: Some(tx),
394            seq: 0,
395        })?;
396        rx.await
397            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
398    }
399
400    /// Seal every batch enqueued **at or before** this call (barrier).
401    ///
402    /// Use after [`Durability::Buffered`] produces to wait for durable PUT (and
403    /// sequencing) of that prior work. Concurrent produces enqueued after this
404    /// call are not required to finish. An empty successful buffer returns immediately.
405    /// A failed PUT or commit remains an error for every barrier covering that
406    /// enqueue prefix, even after later appends succeed. Reopen the engine to
407    /// establish a new prefix from its successfully committed index.
408    pub async fn flush(&self) -> Result<(), ObjectLogError> {
409        let rx = {
410            let mut q = self.shared.queue.lock().expect("poisoned");
411            if q.shutdown {
412                return Err(ObjectLogError::Sequencer("engine shutting down".into()));
413            }
414            if q.next_seq == 1 {
415                // Nothing has ever been enqueued.
416                return Ok(());
417            }
418            let barrier = q.next_seq - 1;
419            if let Some((sequence, error)) = &q.first_failed {
420                if *sequence <= barrier {
421                    return Err(error.clone());
422                }
423            }
424            if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
425                return Ok(());
426            }
427            let (tx, rx) = oneshot::channel();
428            q.force_flush = true;
429            q.flush_waiters.push((barrier, tx));
430            self.shared.cv.notify_all();
431            rx
432        };
433        rx.await
434            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
435    }
436
437    fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
438        let item_len = item.payload.len();
439        let max_buffered_bytes = self.shared.max_buffered_bytes;
440        let shared = Arc::clone(&self.shared);
441        let mut q = shared.queue.lock().expect("poisoned");
442        while !q.shutdown
443            && q.bytes_in_use > 0
444            && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
445        {
446            q = shared.cv.wait(q).expect("poisoned");
447        }
448        if q.shutdown {
449            return Err(ObjectLogError::Sequencer("engine shutting down".into()));
450        }
451        let now = Instant::now();
452        item.seq = q.next_seq;
453        q.next_seq = q.next_seq.saturating_add(1);
454        q.bytes += item_len;
455        q.bytes_in_use += item_len;
456        q.last_enqueue = Some(now);
457        if q.oldest_enqueue.is_none() {
458            q.oldest_enqueue = Some(now);
459        }
460        q.items.push_back(item);
461        shared.cv.notify_all();
462        Ok(())
463    }
464
465    /// Read batches covering offsets at/after `offset`, up to ~`max_bytes`.
466    pub async fn fetch(
467        &self,
468        partition: &PartitionKey,
469        offset: i64,
470        max_bytes: usize,
471    ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
472        let entries = self.sequencer.lookup(partition, offset)?;
473        let mut out = Vec::new();
474        let mut total = 0usize;
475        for e in entries {
476            if total >= max_bytes && !out.is_empty() {
477                break;
478            }
479            let batch = self.load_entry(&e).await?;
480            total += batch.payload.len();
481            out.push(batch);
482        }
483        Ok(out)
484    }
485
486    /// Stream batches from `offset` onward without materializing a full `Vec`.
487    ///
488    /// Calls `visit` once per index entry in order. Suitable for wide offset
489    /// windows (bounded-RAM replay). Stops and returns the error if `visit`
490    /// fails. Unlike [`Self::fetch`], there is no byte budget — visit every remaining
491    /// batch (or stop yourself inside `visit`).
492    pub async fn fetch_stream<F>(
493        &self,
494        partition: &PartitionKey,
495        offset: i64,
496        mut visit: F,
497    ) -> Result<(), ObjectLogError>
498    where
499        F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
500    {
501        let entries = self.sequencer.lookup(partition, offset)?;
502        for e in entries {
503            visit(self.load_entry(&e).await?)?;
504        }
505        Ok(())
506    }
507
508    async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
509        let start = e.location.byte_start as u64;
510        let end = start + e.location.byte_len as u64;
511        let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
512            .await?
513            .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
514        Ok(FetchedBatch {
515            base_offset: e.base_offset,
516            record_count: e.record_count,
517            payload: bytes,
518        })
519    }
520
521    /// Drop the partition's log below `offset` and delete any object that thereby
522    /// becomes fully unreferenced.
523    pub async fn truncate_before(
524        &self,
525        partition: &PartitionKey,
526        offset: i64,
527    ) -> Result<(), ObjectLogError> {
528        let dead = self.sequencer.truncate_before(partition, offset)?;
529        for object_id in dead {
530            self.blob.delete(&object_id).await?;
531        }
532        Ok(())
533    }
534
535    /// Delete data-prefix objects that are not in `live`.
536    ///
537    /// **Safety:** call only when this engine (and any other writer using the
538    /// same prefix) is quiescent — otherwise an in-flight put that has not yet
539    /// been committed may be deleted as an "orphan". Prefer running offline or
540    /// after drop. Does not touch keys outside [`data_prefix`](Self::data_prefix)
541    /// (e.g. a separate manifest prefix is safe).
542    ///
543    /// Returns the deleted object keys.
544    pub async fn reap_orphans(
545        &self,
546        live: &HashSet<String>,
547    ) -> Result<Vec<String>, ObjectLogError> {
548        reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
549    }
550
551    /// Return a point-in-time snapshot of queued and in-flight payload bytes.
552    pub fn buffer_stats(&self) -> BufferStats {
553        let q = self.shared.queue.lock().expect("poisoned");
554        BufferStats {
555            queued_bytes: q.bytes,
556            bytes_in_use: q.bytes_in_use,
557            queued_batches: q.items.len(),
558            max_buffered_bytes: self.shared.max_buffered_bytes,
559        }
560    }
561
562    /// Inspect budget controller layers and counters (TD-004).
563    pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
564        let now = Instant::now();
565        let mut budget = self.shared.budget.lock().expect("poisoned");
566        budget.refill(now);
567        let effective = budget.effective_budget_per_sec;
568        let reason = if !budget.config.enabled {
569            EffectiveReason::Disabled
570        } else if budget.config.budget_per_sec_cap.is_some()
571            && budget
572                .config
573                .budget_per_sec_cap
574                .is_some_and(|c| (effective - c).abs() < 1e-9)
575        {
576            EffectiveReason::ConfigCap
577        } else if budget.ongoing_capacity.is_some() {
578            EffectiveReason::Ongoing
579        } else if budget.startup_capacity.is_some() {
580            EffectiveReason::StartupProbe
581        } else {
582            EffectiveReason::DefaultCapacity
583        };
584        let (queued, last_enq) = {
585            let q = self.shared.queue.lock().expect("poisoned");
586            (q.bytes, q.last_enqueue)
587        };
588        let early = budget.allow_early_flush(now, queued, last_enq);
589        let max_linger = self.shared.flush_config.linger;
590        let eff_linger = if !budget.config.enabled {
591            max_linger
592        } else if early || max_linger.is_zero() {
593            Duration::ZERO
594        } else {
595            max_linger
596        };
597        PipelineSnapshot {
598            budget_per_sec: EffectiveKnob {
599                configured: budget.config.budget_per_sec_cap,
600                startup_measured: budget.startup_capacity,
601                ongoing_measured: budget.ongoing_capacity,
602                effective,
603                reason,
604            },
605            effective_linger_ms: EffectiveKnob {
606                configured: Some(max_linger.as_millis() as u64),
607                startup_measured: None,
608                ongoing_measured: None,
609                effective: eff_linger.as_millis() as u64,
610                reason: if early {
611                    EffectiveReason::Ongoing
612                } else {
613                    EffectiveReason::Configured
614                },
615            },
616            max_linger_ms: max_linger.as_millis() as u64,
617            token_fill_ratio: budget.fill_ratio(),
618            media_ops_total: budget.media_ops_total,
619            overdraft_total: budget.overdraft_total,
620            flushes_total: budget.flushes_total,
621            undersized_deadline_flushes: budget.undersized_deadline_flushes,
622            predicted_media_ops: budget.predicted_media_ops,
623            budget_enabled: budget.config.enabled,
624            budget_mode: budget.config.mode,
625        }
626    }
627}
628
629/// Delete objects under `data_prefix` whose keys are absent from `live`.
630///
631/// See [`LogEngine::reap_orphans`] for safety notes. `live` is typically
632/// [`InMemorySequencer::live_object_ids`](crate::InMemorySequencer::live_object_ids)
633/// or [`ManifestSequencer::live_object_ids`](crate::ManifestSequencer::live_object_ids).
634pub async fn reap_orphans(
635    blob: &dyn BlobStore,
636    data_prefix: &str,
637    live: &HashSet<String>,
638) -> Result<Vec<String>, ObjectLogError> {
639    let keys = blob.list(data_prefix).await?;
640    let mut deleted = Vec::new();
641    for key in keys {
642        if live.contains(&key) {
643            continue;
644        }
645        blob.delete(&key).await?;
646        deleted.push(key);
647    }
648    Ok(deleted)
649}
650
651impl<S: Sequencer> Drop for LogEngine<S> {
652    fn drop(&mut self) {
653        {
654            let mut q = self.shared.queue.lock().expect("poisoned");
655            q.shutdown = true;
656        }
657        self.shared.cv.notify_all();
658        if let Some(t) = self.flush_thread.take() {
659            let _ = t.join();
660        }
661    }
662}
663
664/// Highest numeric suffix already present under `data_prefix` (keys shaped
665/// `{prefix}{counter:020}`). Used so a reopened engine never reissues object ids.
666async fn recover_data_object_counter(
667    blob: &dyn BlobStore,
668    prefix: &str,
669) -> Result<u64, ObjectLogError> {
670    let keys = blob.list(prefix).await?;
671    let mut max = 0u64;
672    for key in keys {
673        let Some(suffix) = key.strip_prefix(prefix) else {
674            continue;
675        };
676        if let Ok(n) = suffix.parse::<u64>() {
677            max = max.max(n);
678        }
679    }
680    Ok(max)
681}
682
683fn flush_loop<S>(
684    shared: Arc<Shared<S::Meta>>,
685    blob: Arc<dyn BlobStore>,
686    sequencer: Arc<S>,
687    config: FlushConfig,
688    prefix: String,
689) where
690    S: Sequencer + 'static,
691    S::Meta: Send + 'static,
692{
693    let max_inflight = config.max_inflight_flushes.max(1);
694    let commit_group_limit = if sequencer.supports_multi_object_commit() {
695        max_inflight
696    } else {
697        1
698    };
699    let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
700        .ok()
701        .and_then(|v| v.parse::<usize>().ok())
702        .filter(|v| *v > 0)
703        .unwrap_or_else(|| max_inflight.min(8));
704    let rt = tokio::runtime::Builder::new_multi_thread()
705        .worker_threads(worker_threads)
706        .enable_all()
707        .build()
708        .expect("flush runtime");
709    // Resume the data-object counter past any keys already under `prefix`. Restarting
710    // at 0 on reopen overwrites sealed objects while manifests still point at the old
711    // byte ranges → RangeOutOfBounds / mid-JSON EOF on fetch (fireweed-481d3e43).
712    let mut counter = match rt.block_on(recover_data_object_counter(blob.as_ref(), &prefix)) {
713        Ok(counter) => counter,
714        Err(error) => {
715            // A failed listing is not an empty prefix. Fail admission before
716            // any PUT can reuse a sealed object's name.
717            let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
718            return;
719        }
720    };
721    let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
722    let mut active_puts = 0usize;
723    let mut committing: Option<CommitJob> = None;
724    let mut shutdown = false;
725
726    loop {
727        while !shutdown && active_puts < max_inflight {
728            // Poll the commit job even with no queued uploads: it owns bytes
729            // whose release may be needed to admit the next producer.
730            let wait_for_more = pending.is_empty() && committing.is_none();
731            match take_batch(&shared, config, wait_for_more) {
732                TakeBatch::Batch(batch) => {
733                    counter += 1;
734                    let concurrent = max_inflight > 1;
735                    let work = start_flush_work(&rt, &blob, &prefix, counter, batch, concurrent);
736                    if concurrent {
737                        active_puts += 1;
738                    }
739                    pending.push_back(work);
740                }
741                TakeBatch::Empty => break,
742                TakeBatch::Shutdown => shutdown = true,
743            }
744        }
745
746        let mut made_progress = false;
747        for work in pending.iter_mut() {
748            let Some(put) = work.put.as_ref() else {
749                continue;
750            };
751            if !put.is_finished() {
752                continue;
753            }
754            let put = work.put.take().expect("put handle exists");
755            let elapsed = work.put_started.elapsed();
756            work.put_result = Some(match rt.block_on(put) {
757                Ok(Ok(())) => Ok(elapsed),
758                Ok(Err(e)) => Err(e),
759                Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
760                    "flush task failed: {e}"
761                ))),
762            });
763            active_puts = active_puts.saturating_sub(1);
764            made_progress = true;
765        }
766
767        if committing
768            .as_ref()
769            .is_some_and(|job| job.task.is_finished())
770        {
771            let job = committing.take().expect("completed commit exists");
772            match rt.block_on(job.task) {
773                Ok(released) => {
774                    let mut q = shared.queue.lock().expect("poisoned");
775                    q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
776                    shared.cv.notify_all();
777                }
778                Err(error) => {
779                    let uploads = fail_engine_and_take_pending_uploads(
780                        &shared,
781                        &mut pending,
782                        job.first_seq,
783                        ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
784                    );
785                    // A shared runtime can outlive this engine. Await accepted uploads
786                    // explicitly so close/reopen/reaping cannot race orphan writes.
787                    // They are never sequenced after the failed commit.
788                    for upload in uploads {
789                        let _ = rt.block_on(upload);
790                    }
791                    return;
792                }
793            }
794            made_progress = true;
795        }
796
797        if committing.is_none() {
798            if let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit) {
799                if max_inflight == 1 {
800                    // Preserve the low-overhead single-flight path.
801                    let released = finish_flush_work(&shared, &blob, &sequencer, work);
802                    let mut q = shared.queue.lock().expect("poisoned");
803                    q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
804                    shared.cv.notify_all();
805                } else {
806                    // One ordered committer; uploads can progress during its
807                    // durable manifest I/O. Bytes remain charged until completion.
808                    let first_seq = work
809                        .batch
810                        .iter()
811                        .map(|p| p.seq)
812                        .min()
813                        .unwrap_or(work.max_seq);
814                    let shared = shared.clone();
815                    let blob = blob.clone();
816                    let sequencer = sequencer.clone();
817                    committing = Some(CommitJob {
818                        task: rt.spawn_blocking(move || {
819                            finish_flush_work(&shared, &blob, &sequencer, work)
820                        }),
821                        first_seq,
822                    });
823                }
824                made_progress = true;
825            }
826        }
827
828        if pending.is_empty() && committing.is_none() {
829            {
830                let mut q = shared.queue.lock().expect("poisoned");
831                if q.items.is_empty() && q.bytes_in_use == 0 {
832                    // Nothing in flight: any remaining waiters with barrier already
833                    // covered (or no work after last seal) can complete.
834                    notify_flush_waiters(&mut q);
835                }
836            }
837            shared.cv.notify_all();
838            if shutdown {
839                // Fail any remaining flush waiters.
840                let mut q = shared.queue.lock().expect("poisoned");
841                for (_, tx) in q.flush_waiters.drain(..) {
842                    let _ = tx.send(Err(ObjectLogError::Sequencer(
843                        "engine shutting down".into(),
844                    )));
845                }
846                return;
847            }
848            continue;
849        }
850
851        if !made_progress {
852            std::thread::sleep(Duration::from_millis(1));
853        }
854    }
855}
856
857/// Close admission and fail queued callers/barriers after an unrecoverable
858/// startup or commit failure. Return started uploads for explicit draining.
859fn fail_engine_and_take_pending_uploads<M>(
860    shared: &Arc<Shared<M>>,
861    pending: &mut VecDeque<FlushWork<M>>,
862    first_seq: u64,
863    error: ObjectLogError,
864) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
865    let mut uploads = Vec::new();
866    for mut work in pending.drain(..) {
867        if let Some(put) = work.put.take() {
868            uploads.push(put);
869        }
870        send_storage_error(&mut work.responders, error.clone());
871    }
872    // Recover a poisoned guard only to close the engine, never to resume writes.
873    let mut q = shared
874        .queue
875        .lock()
876        .unwrap_or_else(|poisoned| poisoned.into_inner());
877    q.shutdown = true;
878    q.first_failed.get_or_insert((first_seq, error.clone()));
879    for mut item in q.items.drain(..) {
880        if let Some(tx) = item.responder.take() {
881            let _ = tx.send(Err(error.clone()));
882        }
883    }
884    q.bytes = 0;
885    // Keep outstanding bytes conservatively charged until the failed engine
886    // and its accepted uploads have completed.
887    for (_, tx) in q.flush_waiters.drain(..) {
888        let _ = tx.send(Err(error.clone()));
889    }
890    q.force_flush = false;
891    shared.cv.notify_all();
892    uploads
893}
894
895/// Combine only an already-ready contiguous success prefix. Never wait to fill
896/// a group or cross an unfinished/failed PUT; preserve object and batch order.
897fn take_ready_commit_group<M>(
898    pending: &mut VecDeque<FlushWork<M>>,
899    max_objects: usize,
900) -> Option<FlushWork<M>> {
901    pending.front()?.put_result.as_ref()?;
902    let mut group = pending.pop_front().expect("ready front exists");
903    if group.put_result.as_ref().is_some_and(Result::is_err) {
904        return Some(group);
905    }
906    while group.data_objects < max_objects.max(1) {
907        let Some(next) = pending.front() else { break };
908        if !next.put_result.as_ref().is_some_and(Result::is_ok)
909            || group.data_objects.saturating_add(next.data_objects) > max_objects
910        {
911            break;
912        }
913        let mut next = pending.pop_front().expect("ready successor exists");
914        group.batch.append(&mut next.batch);
915        group.locations.append(&mut next.locations);
916        group.responders.append(&mut next.responders);
917        group.bytes = group
918            .bytes
919            .checked_add(next.bytes)
920            .expect("bounded pending bytes");
921        group.max_seq = group.max_seq.max(next.max_seq);
922        group.data_objects += next.data_objects;
923        let next_elapsed = next.put_result.take().unwrap().unwrap();
924        let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
925        *elapsed = (*elapsed).max(next_elapsed);
926    }
927    Some(group)
928}
929
930/// How long to sleep before re-evaluating a seal decision.
931/// - `ZERO` means seal now (early-flush or linger deadline hit).
932/// - Otherwise wait at most this long (interrupted by new produces).
933fn effective_linger<M>(
934    shared: &Shared<M>,
935    config: FlushConfig,
936    queued_bytes: usize,
937    last_enqueue: Option<Instant>,
938    oldest_enqueue: Option<Instant>,
939) -> Duration {
940    if config.linger.is_zero() {
941        return Duration::ZERO;
942    }
943    let now = Instant::now();
944    if config.budget.enabled {
945        let mut budget = shared.budget.lock().expect("poisoned");
946        budget.refill(now);
947        if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
948            budget.note_early_flush(now);
949            return Duration::ZERO;
950        }
951    }
952    // True linger: seal when the oldest buffered item reaches max wait.
953    if let Some(oldest) = oldest_enqueue {
954        let deadline = oldest + config.linger;
955        if now >= deadline {
956            return Duration::ZERO;
957        }
958        let until_deadline = deadline.saturating_duration_since(now);
959        // Also wake when idle gate could open, so sparse produces don't wait full linger.
960        if config.budget.enabled
961            && !config.budget.early_flush_idle.is_zero()
962            && let Some(last) = last_enqueue
963        {
964            let idle_at = last + config.budget.early_flush_idle;
965            if idle_at > now && idle_at < deadline {
966                return idle_at.saturating_duration_since(now);
967            }
968        }
969        return until_deadline;
970    }
971    config.linger
972}
973
974fn take_batch<M>(
975    shared: &Arc<Shared<M>>,
976    config: FlushConfig,
977    wait_when_empty: bool,
978) -> TakeBatch<M> {
979    let mut q = shared.queue.lock().expect("poisoned");
980    loop {
981        if q.items.is_empty() {
982            if q.shutdown {
983                return TakeBatch::Shutdown;
984            }
985            if !wait_when_empty {
986                return TakeBatch::Empty;
987            }
988            // No pending work needs a timer. Enqueue and shutdown change the
989            // predicate under this mutex and notify. Waiting without
990            // dropping/reacquiring it first also avoids a lost notification.
991            q = shared.cv.wait(q).expect("poisoned");
992            continue;
993        }
994
995        let linger = {
996            let queued = q.bytes;
997            let last_enq = q.last_enqueue;
998            let oldest = q.oldest_enqueue;
999            drop(q);
1000            let l = effective_linger(shared, config, queued, last_enq, oldest);
1001            q = shared.queue.lock().expect("poisoned");
1002            l
1003        };
1004
1005        let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1006        let force = q.force_flush;
1007        let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1008        if triggered {
1009            if !size_trigger && !q.shutdown && !q.items.is_empty() {
1010                let mut budget = shared.budget.lock().expect("poisoned");
1011                if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1012                    budget.undersized_deadline_flushes += 1;
1013                }
1014            }
1015            break;
1016        }
1017
1018        // Short wait (until idle gate or linger deadline); re-evaluate, do not
1019        // seal solely because a probe sleep timed out.
1020        let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1021        q = guard;
1022    }
1023
1024    let force_drain = q.force_flush;
1025    let mut items = Vec::new();
1026    let mut bytes = 0usize;
1027    // On force_flush, still respect max_bytes so one object stays bounded, but
1028    // ignore max_batches so a barrier can drain large queues across objects.
1029    while let Some(item) = q.items.pop_front() {
1030        q.bytes = q.bytes.saturating_sub(item.payload.len());
1031        bytes += item.payload.len();
1032        items.push(item);
1033        if bytes >= config.max_bytes {
1034            break;
1035        }
1036        if !force_drain && items.len() >= config.max_batches {
1037            break;
1038        }
1039    }
1040    q.oldest_enqueue = if q.items.is_empty() {
1041        None
1042    } else {
1043        // Approximate: next seal window starts now for remaining items.
1044        Some(Instant::now())
1045    };
1046    // Keep forcing while a barrier still has queued work.
1047    if force_drain {
1048        q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1049    }
1050    if items.is_empty() {
1051        TakeBatch::Empty
1052    } else {
1053        TakeBatch::Batch(items)
1054    }
1055}
1056
1057fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1058    let done = q.completed_through;
1059    let mut i = 0;
1060    while i < q.flush_waiters.len() {
1061        let barrier = q.flush_waiters[i].0;
1062        let failure = q
1063            .first_failed
1064            .as_ref()
1065            .filter(|(sequence, _)| *sequence <= barrier)
1066            .map(|(_, error)| error.clone());
1067        if barrier <= done || failure.is_some() {
1068            let (_, tx) = q.flush_waiters.swap_remove(i);
1069            let _ = tx.send(failure.map_or(Ok(()), Err));
1070        } else {
1071            i += 1;
1072        }
1073    }
1074    if q.flush_waiters.is_empty() {
1075        q.force_flush = false;
1076    } else if q.items.is_empty() && q.bytes_in_use == 0 {
1077        // Barriers still waiting but no work — should not happen; clear force.
1078        q.force_flush = true;
1079    } else {
1080        q.force_flush = true;
1081    }
1082}
1083
1084fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1085    for (_, tx) in responders.iter_mut() {
1086        if let Some(tx) = tx.take() {
1087            let _ = tx.send(Err(err.clone()));
1088        }
1089    }
1090}
1091
1092fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1093    for (durability, tx) in responders.iter_mut() {
1094        if *durability == Durability::Durable
1095            && let Some(tx) = tx.take()
1096        {
1097            let _ = tx.send(Ok(AppendOutcome {
1098                base_offset: None,
1099                last_offset: None,
1100                durable: true,
1101                sequenced: false,
1102            }));
1103        }
1104    }
1105}
1106
1107fn prepare_flush_work<M>(
1108    prefix: &str,
1109    counter: u64,
1110    mut batch: Vec<Pending<M>>,
1111) -> (FlushWork<M>, String, Vec<Bytes>) {
1112    let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1113    let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1114    let key = format!("{prefix}{counter:020}");
1115    let mut offset = 0usize;
1116    for p in &batch {
1117        let start = offset as u32;
1118        offset += p.payload.len();
1119        chunks.push(p.payload.clone());
1120        locations.push(BatchLocation {
1121            object_id: key.clone(),
1122            byte_start: start,
1123            byte_len: p.payload.len() as u32,
1124        });
1125    }
1126    if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1127        eprintln!(
1128            "object-log seal: key={key} batches={} bytes={offset}",
1129            batch.len()
1130        );
1131    }
1132    let responders: Vec<(Durability, Option<Responder>)> = batch
1133        .iter_mut()
1134        .map(|p| (p.durability, p.responder.take()))
1135        .collect();
1136    let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1137    let work = FlushWork {
1138        batch,
1139        locations,
1140        responders,
1141        bytes: offset,
1142        max_seq,
1143        data_objects: 1,
1144        put: None,
1145        put_started: Instant::now(),
1146        put_result: None,
1147    };
1148    (work, key, chunks)
1149}
1150
1151/// Start a durable put for a sealed batch.
1152///
1153/// - `concurrent == false` (default `max_inflight_flushes == 1`): `block_on` the
1154///   put on the flush thread; Local uses `block_in_place` (no spawn_blocking queue).
1155/// - `concurrent == true`: spawn put tasks for parallel S3-style throughput.
1156fn start_flush_work<M>(
1157    rt: &tokio::runtime::Runtime,
1158    blob: &Arc<dyn BlobStore>,
1159    prefix: &str,
1160    counter: u64,
1161    batch: Vec<Pending<M>>,
1162    concurrent: bool,
1163) -> FlushWork<M> {
1164    let (mut work, key, chunks) = prepare_flush_work(prefix, counter, batch);
1165    let blob = Arc::clone(blob);
1166    let _ = blob.take_media_op_stats();
1167    work.put_started = Instant::now();
1168    if concurrent {
1169        work.put =
1170            Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
1171    } else {
1172        let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1173        let elapsed = work.put_started.elapsed();
1174        work.put_result = Some(match put_result {
1175            Ok(()) => Ok(elapsed),
1176            Err(e) => Err(e),
1177        });
1178    }
1179    work
1180}
1181
1182fn finish_flush_work<S, M>(
1183    shared: &Arc<Shared<M>>,
1184    blob: &Arc<dyn BlobStore>,
1185    sequencer: &Arc<S>,
1186    mut work: FlushWork<S::Meta>,
1187) -> usize
1188where
1189    S: Sequencer + 'static,
1190    S::Meta: Send + 'static,
1191{
1192    let release_bytes = work.bytes;
1193    let first_seq = work
1194        .batch
1195        .iter()
1196        .map(|pending| pending.seq)
1197        .min()
1198        .unwrap_or(work.max_seq);
1199    // Durable-then-sequence: the object PUT may have overlapped later PUTs, but
1200    // sequencer commits are still completed in object creation order.
1201    // Grouped put_ms is the largest observed member PUT duration, not a sum.
1202    let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1203    let put_elapsed = match work.put_result.take().expect("put result is ready") {
1204        Ok(elapsed) => elapsed,
1205        Err(e) => {
1206            send_storage_error(&mut work.responders, e.clone());
1207            let mut q = shared.queue.lock().expect("poisoned");
1208            // Unblock flush waiters covering this seq with the storage error.
1209            let done = work.max_seq;
1210            if done >= q.completed_through {
1211                q.completed_through = done;
1212            }
1213            q.first_failed.get_or_insert((first_seq, e));
1214            notify_flush_waiters(&mut q);
1215            shared.cv.notify_all();
1216            return release_bytes;
1217        }
1218    };
1219
1220    // Media ops for the data object put.
1221    let mut media_ops = blob
1222        .take_media_op_stats()
1223        .map(|s| s.media_ops)
1224        .unwrap_or(work.data_objects as u64); // fallback: 1 per successful put
1225
1226    // Signal Durable-level waiters now (after PUT, before commit).
1227    send_durable_acks(&mut work.responders);
1228
1229    // Sequence the ready group atomically, preserving each object's locations.
1230    let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1231        .batch
1232        .iter()
1233        .zip(work.locations.iter())
1234        .map(|(p, loc)| CommitBatch {
1235            partition: p.partition.clone(),
1236            record_count: p.record_count,
1237            location: loc.clone(),
1238            meta: &p.meta,
1239        })
1240        .collect();
1241
1242    let commit_started = timing.then(Instant::now);
1243    // Clear stats again so sequencer durable work (e.g. ManifestSequencer put)
1244    // on the shared store is counted into the same budget.
1245    let _ = blob.take_media_op_stats();
1246    let mut commit_error = None;
1247    match sequencer.commit(&commit_batches) {
1248        Ok(outcomes) => {
1249            if let Some(commit_started) = commit_started {
1250                eprintln!(
1251                    "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1252                    work.bytes,
1253                    commit_batches.len(),
1254                    put_elapsed.as_millis(),
1255                    commit_started.elapsed().as_millis(),
1256                    work.data_objects
1257                );
1258            }
1259            if let Some(s) = blob.take_media_op_stats() {
1260                media_ops = media_ops.saturating_add(s.media_ops);
1261            }
1262            for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1263                if let Some(tx) = tx.take() {
1264                    let (base, last) = match outcome {
1265                        CommitOutcome::Assigned {
1266                            base_offset,
1267                            record_count,
1268                        } => (
1269                            Some(base_offset),
1270                            Some(base_offset + record_count as i64 - 1),
1271                        ),
1272                        CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1273                    };
1274                    let _ = tx.send(Ok(AppendOutcome {
1275                        base_offset: base,
1276                        last_offset: last,
1277                        durable: true,
1278                        sequenced: true,
1279                    }));
1280                }
1281            }
1282        }
1283        Err(e) => {
1284            if let Some(s) = blob.take_media_op_stats() {
1285                media_ops = media_ops.saturating_add(s.media_ops);
1286            }
1287            send_storage_error(&mut work.responders, e.clone());
1288            commit_error = Some(e);
1289        }
1290    }
1291
1292    {
1293        let mut budget = shared.budget.lock().expect("poisoned");
1294        budget.consume_after_flush(media_ops, Instant::now());
1295    }
1296    {
1297        let mut q = shared.queue.lock().expect("poisoned");
1298        if let Some(error) = commit_error {
1299            q.first_failed.get_or_insert((first_seq, error));
1300        }
1301        if work.max_seq >= q.completed_through {
1302            q.completed_through = work.max_seq;
1303        }
1304        notify_flush_waiters(&mut q);
1305    }
1306    shared.cv.notify_all();
1307
1308    release_bytes
1309}
1310
1311#[cfg(test)]
1312mod flush_barrier_tests {
1313    use super::*;
1314
1315    #[test]
1316    fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1317        let error = ObjectLogError::Sequencer("manifest failed".into());
1318        let mut queue = Queue::<()> {
1319            items: VecDeque::new(),
1320            bytes: 0,
1321            bytes_in_use: 1,
1322            shutdown: false,
1323            next_seq: 4,
1324            completed_through: 2,
1325            first_failed: Some((2, error.clone())),
1326            force_flush: true,
1327            flush_waiters: Vec::new(),
1328            last_enqueue: None,
1329            oldest_enqueue: None,
1330        };
1331        let mut receivers = Vec::new();
1332        for barrier in [1, 2, 3] {
1333            let (tx, rx) = oneshot::channel();
1334            queue.flush_waiters.push((barrier, tx));
1335            receivers.push(rx);
1336        }
1337        notify_flush_waiters(&mut queue);
1338        assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1339        assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1340        assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1341        assert!(queue.flush_waiters.is_empty());
1342    }
1343}
1344
1345#[cfg(test)]
1346mod ready_commit_group_tests {
1347    use super::*;
1348    use crate::{ManifestSequencer, MemoryBlobStore};
1349    use async_trait::async_trait;
1350    use std::ops::Range;
1351
1352    fn work(
1353        n: u64,
1354        durability: Durability,
1355    ) -> (
1356        FlushWork<()>,
1357        String,
1358        Vec<Bytes>,
1359        oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1360    ) {
1361        let (tx, rx) = oneshot::channel();
1362        let (mut work, key, chunks) = prepare_flush_work(
1363            "data/",
1364            n,
1365            vec![Pending {
1366                partition: PartitionKey("shared".into()),
1367                record_count: 1,
1368                payload: Bytes::from(format!("body-{n}")),
1369                meta: (),
1370                durability,
1371                responder: Some(tx),
1372                seq: n,
1373            }],
1374        );
1375        work.put_result = Some(Ok(Duration::from_millis(n)));
1376        (work, key, chunks, rx)
1377    }
1378
1379    #[test]
1380    fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1381        let mut pending = (1..=5)
1382            .map(|n| work(n, Durability::Sequenced).0)
1383            .collect::<VecDeque<_>>();
1384        let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1385        let mut released = 0;
1386        for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1387            let group = take_ready_commit_group(&mut pending, 2).unwrap();
1388            assert_eq!(
1389                group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1390                expected
1391            );
1392            assert_eq!(group.data_objects, expected.len());
1393            assert_eq!(group.responders.len(), expected.len());
1394            assert_eq!(group.max_seq, *expected.last().unwrap());
1395            for (location, n) in group.locations.iter().zip(&expected) {
1396                assert_eq!(location.object_id, format!("data/{n:020}"));
1397                assert_eq!(location.byte_start, 0);
1398                assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1399            }
1400            released += group.bytes;
1401        }
1402        assert!(pending.is_empty());
1403        assert_eq!(released, original_bytes);
1404
1405        let mut first = work(1, Durability::Sequenced).0;
1406        first.put_result = None;
1407        let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1408        assert!(take_ready_commit_group(&mut pending, 8).is_none());
1409        assert_eq!(
1410            pending.len(),
1411            2,
1412            "a later ready PUT cannot pass an unfinished head"
1413        );
1414        pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1415        let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1416        assert_eq!(failed.data_objects, 1);
1417        assert!(failed.put_result.unwrap().is_err());
1418        assert_eq!(pending[0].max_seq, 2);
1419
1420        let mut failed = work(2, Durability::Sequenced).0;
1421        failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1422        let mut pending = VecDeque::from([
1423            work(1, Durability::Sequenced).0,
1424            failed,
1425            work(3, Durability::Sequenced).0,
1426        ]);
1427        assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1428        assert_eq!(
1429            pending.len(),
1430            2,
1431            "a success group cannot cross a failed PUT"
1432        );
1433        let mut pending = VecDeque::from([
1434            work(1, Durability::Sequenced).0,
1435            work(2, Durability::Sequenced).0,
1436        ]);
1437        assert_eq!(
1438            take_ready_commit_group(&mut pending, 1)
1439                .unwrap()
1440                .data_objects,
1441            1
1442        );
1443        assert_eq!(
1444            pending.len(),
1445            1,
1446            "one-object sequencers retain their call boundary"
1447        );
1448    }
1449
1450    struct GatedManifestStore {
1451        inner: MemoryBlobStore,
1452        entered: std::sync::mpsc::SyncSender<()>,
1453        release: Mutex<Option<oneshot::Receiver<()>>>,
1454        fail: bool,
1455        panic_on_commit: bool,
1456    }
1457
1458    #[async_trait]
1459    impl BlobStore for GatedManifestStore {
1460        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1461            if key.starts_with("manifest/") {
1462                let release = self.release.lock().unwrap().take();
1463                if let Some(release) = release {
1464                    self.entered.send(()).unwrap();
1465                    release.await.unwrap();
1466                    assert!(!self.panic_on_commit, "injected commit panic");
1467                    if self.fail {
1468                        return Err(ObjectLogError::StorageUnavailable(
1469                            "manifest failure".into(),
1470                        ));
1471                    }
1472                }
1473            }
1474            self.inner.put(key, value).await
1475        }
1476        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1477            self.inner.get(key).await
1478        }
1479        async fn get_range(
1480            &self,
1481            key: &str,
1482            range: Range<u64>,
1483        ) -> Result<Option<Bytes>, ObjectLogError> {
1484            self.inner.get_range(key, range).await
1485        }
1486        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1487            self.inner.list(prefix).await
1488        }
1489        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1490            self.inner.delete(key).await
1491        }
1492        fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1493            self.inner.take_media_op_stats()
1494        }
1495    }
1496
1497    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1498    async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1499        for fail in [false, true] {
1500            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1501            let (release_tx, release_rx) = oneshot::channel();
1502            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1503                inner: MemoryBlobStore::new(),
1504                entered: entered_tx,
1505                release: Mutex::new(Some(release_rx)),
1506                fail,
1507                panic_on_commit: false,
1508            });
1509            let sequencer = Arc::new(
1510                ManifestSequencer::open(blob.clone(), "manifest/")
1511                    .await
1512                    .unwrap(),
1513            );
1514            assert!(sequencer.supports_multi_object_commit());
1515            let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1516            let (mut second, second_key, second_chunks, mut durable_rx) =
1517                work(2, Durability::Durable);
1518            second.batch[0].partition = PartitionKey("other".into());
1519            let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1520            blob.put_chunks(&first_key, first_chunks).await.unwrap();
1521            blob.put_chunks(&second_key, second_chunks).await.unwrap();
1522            blob.put_chunks(&third_key, third_chunks).await.unwrap();
1523            let mut pending = VecDeque::from([first, second, third]);
1524            let group = take_ready_commit_group(&mut pending, 8).unwrap();
1525            let bytes = group.bytes;
1526            assert_eq!(group.data_objects, 3);
1527            let (flush_tx, mut flush_rx) = oneshot::channel();
1528            let mut config = FlushConfig::default();
1529            config.budget.enabled = false;
1530            let shared: Arc<Shared<()>> = Arc::new(Shared {
1531                queue: Mutex::new(Queue {
1532                    items: VecDeque::new(),
1533                    bytes: 0,
1534                    bytes_in_use: bytes,
1535                    shutdown: false,
1536                    next_seq: 4,
1537                    completed_through: 0,
1538                    first_failed: None,
1539                    force_flush: true,
1540                    flush_waiters: vec![(3, flush_tx)],
1541                    last_enqueue: None,
1542                    oldest_enqueue: None,
1543                }),
1544                cv: Condvar::new(),
1545                max_buffered_bytes: bytes,
1546                budget: Mutex::new(BudgetRuntime::new(config.budget)),
1547                flush_config: config,
1548            });
1549            let worker = {
1550                let shared = shared.clone();
1551                let blob = blob.clone();
1552                let sequencer = sequencer.clone();
1553                std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1554            };
1555            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1556            assert!(matches!(
1557                sequenced_rx.try_recv(),
1558                Err(oneshot::error::TryRecvError::Empty)
1559            ));
1560            assert!(matches!(
1561                third_rx.try_recv(),
1562                Err(oneshot::error::TryRecvError::Empty)
1563            ));
1564            assert!(matches!(
1565                flush_rx.try_recv(),
1566                Err(oneshot::error::TryRecvError::Empty)
1567            ));
1568            let durable = durable_rx.try_recv().unwrap().unwrap();
1569            assert!(durable.durable && !durable.sequenced);
1570            release_tx.send(()).unwrap();
1571            assert_eq!(worker.join().unwrap(), bytes);
1572            if fail {
1573                assert!(sequenced_rx.await.unwrap().is_err());
1574                assert!(third_rx.await.unwrap().is_err());
1575                assert!(flush_rx.await.unwrap().is_err());
1576                assert_eq!(
1577                    sequencer
1578                        .high_watermark(&PartitionKey("shared".into()))
1579                        .unwrap(),
1580                    0
1581                );
1582                assert!(blob.list("manifest/").await.unwrap().is_empty());
1583            } else {
1584                assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1585                assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1586                flush_rx.await.unwrap().unwrap();
1587                assert_eq!(blob.list("manifest/").await.unwrap().len(), 1);
1588            }
1589            assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1590            let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1591                .await
1592                .unwrap();
1593            for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1594                let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1595                assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1596                for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1597                    assert_eq!(entry.base_offset, offset as i64);
1598                    let loc = &entry.location;
1599                    assert_eq!(loc.object_id, format!("data/{object:020}"));
1600                    let body = blob
1601                        .get_range(
1602                            &loc.object_id,
1603                            loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1604                        )
1605                        .await
1606                        .unwrap()
1607                        .unwrap();
1608                    assert_eq!(body, format!("body-{object}"));
1609                }
1610            }
1611        }
1612    }
1613
1614    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1615    async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1616        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1617        let (release_tx, release_rx) = oneshot::channel();
1618        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1619            inner: MemoryBlobStore::new(),
1620            entered: entered_tx,
1621            release: Mutex::new(Some(release_rx)),
1622            fail: false,
1623            panic_on_commit: false,
1624        });
1625        let sequencer = Arc::new(
1626            ManifestSequencer::open(blob.clone(), "manifest/")
1627                .await
1628                .unwrap(),
1629        );
1630        let mut config = FlushConfig::default();
1631        config.max_batches = 1;
1632        config.max_inflight_flushes = 4;
1633        config.linger = Duration::ZERO;
1634        config.budget.enabled = false;
1635        let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1636        let first = {
1637            let engine = engine.clone();
1638            tokio::spawn(async move {
1639                engine
1640                    .produce(
1641                        PartitionKey("p".into()),
1642                        Bytes::from_static(b"first"),
1643                        1,
1644                        (),
1645                        Durability::Sequenced,
1646                    )
1647                    .await
1648            })
1649        };
1650        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1651        for n in 1..4 {
1652            engine
1653                .produce(
1654                    PartitionKey("p".into()),
1655                    Bytes::from(format!("later-{n}")),
1656                    1,
1657                    (),
1658                    Durability::Buffered,
1659                )
1660                .await
1661                .unwrap();
1662        }
1663        let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1664            loop {
1665                if blob.list("data/").await.unwrap().len() == 4 {
1666                    break;
1667                }
1668                tokio::time::sleep(Duration::from_millis(1)).await;
1669            }
1670        })
1671        .await
1672        .is_ok();
1673        let early_ack = first.is_finished();
1674        // Always release before assertions so a failed regression can drain.
1675        release_tx.send(()).unwrap();
1676        first.await.unwrap().unwrap();
1677        engine.flush().await.unwrap();
1678        let rows = engine
1679            .fetch(&PartitionKey("p".into()), 0, 1024)
1680            .await
1681            .unwrap();
1682        assert_eq!(
1683            rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
1684            vec![0, 1, 2, 3]
1685        );
1686        drop(engine);
1687        assert!(
1688            !early_ack,
1689            "sequenced acknowledgement preceded manifest durability"
1690        );
1691        assert!(
1692            upload_progress,
1693            "blocked manifest prevented independent data uploads"
1694        );
1695    }
1696
1697    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1698    async fn commit_completion_releases_admission_bytes_without_more_uploads() {
1699        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1700        let sequencer = Arc::new(
1701            ManifestSequencer::open(blob.clone(), "manifest/")
1702                .await
1703                .unwrap(),
1704        );
1705        let mut config = FlushConfig::default();
1706        config.max_bytes = 4;
1707        config.max_buffered_bytes = 4;
1708        config.max_batches = 1;
1709        config.max_inflight_flushes = 4;
1710        config.linger = Duration::ZERO;
1711        config.budget.enabled = false;
1712        let engine = LogEngine::new(blob, sequencer, config, "data/");
1713        engine
1714            .produce(
1715                PartitionKey("p".into()),
1716                Bytes::from_static(b"full"),
1717                1,
1718                (),
1719                Durability::Sequenced,
1720            )
1721            .await
1722            .unwrap();
1723        tokio::time::timeout(Duration::from_secs(5), async {
1724            while engine.buffer_stats().bytes_in_use != 0 {
1725                tokio::time::sleep(Duration::from_millis(1)).await;
1726            }
1727        })
1728        .await
1729        .expect("completed committer stranded admission bytes");
1730        engine
1731            .produce(
1732                PartitionKey("p".into()),
1733                Bytes::from_static(b"next"),
1734                1,
1735                (),
1736                Durability::Sequenced,
1737            )
1738            .await
1739            .unwrap();
1740        engine.flush().await.unwrap();
1741        assert_eq!(
1742            engine
1743                .fetch(&PartitionKey("p".into()), 0, 100)
1744                .await
1745                .unwrap()
1746                .len(),
1747            2
1748        );
1749    }
1750
1751    struct FailureInjectingStore {
1752        inner: Arc<dyn BlobStore>,
1753        second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
1754        second_upload_completed: std::sync::atomic::AtomicBool,
1755        fail_data_listing: std::sync::atomic::AtomicBool,
1756    }
1757
1758    impl FailureInjectingStore {
1759        fn new(inner: Arc<dyn BlobStore>) -> Self {
1760            Self {
1761                inner,
1762                second_upload_gate: Mutex::new(None),
1763                second_upload_completed: std::sync::atomic::AtomicBool::new(false),
1764                fail_data_listing: std::sync::atomic::AtomicBool::new(false),
1765            }
1766        }
1767    }
1768
1769    #[async_trait]
1770    impl BlobStore for FailureInjectingStore {
1771        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1772            let gate = if key == "data/00000000000000000002" {
1773                self.second_upload_gate.lock().unwrap().take()
1774            } else {
1775                None
1776            };
1777            if let Some((entered, release)) = gate {
1778                entered.send(()).unwrap();
1779                release.await.unwrap();
1780                let result = self.inner.put(key, value).await;
1781                if result.is_ok() {
1782                    self.second_upload_completed
1783                        .store(true, std::sync::atomic::Ordering::SeqCst);
1784                }
1785                return result;
1786            }
1787            self.inner.put(key, value).await
1788        }
1789        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1790            self.inner.get(key).await
1791        }
1792        async fn get_range(
1793            &self,
1794            key: &str,
1795            range: Range<u64>,
1796        ) -> Result<Option<Bytes>, ObjectLogError> {
1797            self.inner.get_range(key, range).await
1798        }
1799        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1800            if prefix == "data/"
1801                && self
1802                    .fail_data_listing
1803                    .load(std::sync::atomic::Ordering::SeqCst)
1804            {
1805                return Err(ObjectLogError::StorageUnavailable(
1806                    "injected object listing failure".into(),
1807                ));
1808            }
1809            self.inner.list(prefix).await
1810        }
1811        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1812            self.inner.delete(key).await
1813        }
1814    }
1815
1816    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1817    async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
1818        let inner = Arc::new(MemoryBlobStore::new());
1819        let key = "data/00000000000000000001";
1820        let original = Bytes::from_static(b"existing immutable object");
1821        inner.put(key, original.clone()).await.unwrap();
1822        let store = Arc::new(FailureInjectingStore::new(inner.clone()));
1823        store
1824            .fail_data_listing
1825            .store(true, std::sync::atomic::Ordering::SeqCst);
1826        let mut config = FlushConfig::default();
1827        config.max_batches = 1;
1828        config.max_inflight_flushes = 4;
1829        config.linger = Duration::ZERO;
1830        config.budget.enabled = false;
1831        let engine = LogEngine::new(
1832            store,
1833            Arc::new(crate::InMemorySequencer::new()),
1834            config,
1835            "data/",
1836        );
1837        let result = tokio::time::timeout(
1838            Duration::from_secs(5),
1839            engine.produce(
1840                PartitionKey("p".into()),
1841                Bytes::from_static(b"must not overwrite"),
1842                1,
1843                (),
1844                Durability::Sequenced,
1845            ),
1846        )
1847        .await;
1848        let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
1849        tokio::task::spawn_blocking(move || drop(engine))
1850            .await
1851            .unwrap();
1852        let retained = inner.get(key).await.unwrap();
1853        assert_eq!(
1854            retained,
1855            Some(original),
1856            "failed listing must preserve sealed bytes"
1857        );
1858        assert!(
1859            result.unwrap().is_err(),
1860            "failed prefix recovery must reject admission"
1861        );
1862        assert!(
1863            flushed.unwrap().is_err(),
1864            "flush must report failed initialization"
1865        );
1866        assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
1867    }
1868
1869    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1870    async fn engines_commit_and_close_independently() {
1871        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1872        let (release_tx, release_rx) = oneshot::channel();
1873        let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1874            inner: MemoryBlobStore::new(),
1875            entered: entered_tx,
1876            release: Mutex::new(Some(release_rx)),
1877            fail: false,
1878            panic_on_commit: false,
1879        })));
1880        let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
1881        let mut config = FlushConfig::default();
1882        config.max_batches = 1;
1883        config.max_inflight_flushes = 4;
1884        config.linger = Duration::ZERO;
1885        config.budget.enabled = false;
1886        let a_seq = Arc::new(
1887            ManifestSequencer::open(a_blob.clone(), "manifest/")
1888                .await
1889                .unwrap(),
1890        );
1891        let b_seq = Arc::new(
1892            ManifestSequencer::open(b_blob.clone(), "manifest/")
1893                .await
1894                .unwrap(),
1895        );
1896        let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
1897        a.produce(
1898            PartitionKey("p".into()),
1899            Bytes::from_static(b"a-first"),
1900            1,
1901            (),
1902            Durability::Buffered,
1903        )
1904        .await
1905        .unwrap();
1906        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1907        let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
1908        let b_result = tokio::time::timeout(
1909            Duration::from_secs(5),
1910            b.produce(
1911                PartitionKey("p".into()),
1912                Bytes::from_static(b"b-first"),
1913                1,
1914                (),
1915                Durability::Sequenced,
1916            ),
1917        )
1918        .await;
1919        let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
1920        let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
1921        // Always release the held manifest before assertions, including the pre-change red run.
1922        release_tx.send(()).unwrap();
1923        if b_closed.is_err() {
1924            closing_b.await.unwrap();
1925        }
1926        assert!(
1927            b_closed.is_ok(),
1928            "closing B waited for A's blocked manifest"
1929        );
1930        assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
1931        a.flush().await.unwrap();
1932        assert_eq!(
1933            a.produce(
1934                PartitionKey("p".into()),
1935                Bytes::from_static(b"a-second"),
1936                1,
1937                (),
1938                Durability::Sequenced
1939            )
1940            .await
1941            .unwrap()
1942            .base_offset,
1943            Some(1)
1944        );
1945        let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
1946        assert_eq!(
1947            rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
1948            vec![
1949                Bytes::from_static(b"a-first"),
1950                Bytes::from_static(b"a-second")
1951            ]
1952        );
1953        tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
1954    }
1955
1956    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1957    async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
1958        let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
1959        let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
1960        let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
1961        let (upload_release_tx, upload_release_rx) = oneshot::channel();
1962        let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1963            inner: MemoryBlobStore::new(),
1964            entered: manifest_entered_tx,
1965            release: Mutex::new(Some(manifest_release_rx)),
1966            fail: false,
1967            panic_on_commit: true,
1968        })));
1969        *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
1970        let mut config = FlushConfig::default();
1971        config.max_batches = 1;
1972        config.max_inflight_flushes = 4;
1973        config.linger = Duration::ZERO;
1974        config.budget.enabled = false;
1975        let sequencer = Arc::new(
1976            ManifestSequencer::open(store.clone(), "manifest/")
1977                .await
1978                .unwrap(),
1979        );
1980        let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
1981        engine
1982            .produce(
1983                PartitionKey("p".into()),
1984                Bytes::from_static(b"first"),
1985                1,
1986                (),
1987                Durability::Buffered,
1988            )
1989            .await
1990            .unwrap();
1991        manifest_entered_rx
1992            .recv_timeout(Duration::from_secs(5))
1993            .unwrap();
1994        engine
1995            .produce(
1996                PartitionKey("p".into()),
1997                Bytes::from_static(b"second"),
1998                1,
1999                (),
2000                Durability::Buffered,
2001            )
2002            .await
2003            .unwrap();
2004        upload_entered_rx
2005            .recv_timeout(Duration::from_secs(5))
2006            .unwrap();
2007        manifest_release_tx.send(()).unwrap();
2008        tokio::time::timeout(Duration::from_secs(5), async {
2009            while !engine.shared.queue.lock().unwrap().shutdown {
2010                tokio::time::sleep(Duration::from_millis(1)).await;
2011            }
2012        })
2013        .await
2014        .unwrap();
2015        let flush_failed = engine.flush().await.is_err();
2016        // A failed committer closes admission first; its accepted PUT must still
2017        // complete before close returns, even though no manifest may reference it.
2018        let upload_was_owned = upload_release_tx.send(()).is_ok();
2019        tokio::time::timeout(
2020            Duration::from_secs(5),
2021            tokio::task::spawn_blocking(move || drop(engine)),
2022        )
2023        .await
2024        .unwrap()
2025        .unwrap();
2026        assert!(flush_failed);
2027        assert!(upload_was_owned);
2028        assert!(
2029            store
2030                .second_upload_completed
2031                .load(std::sync::atomic::Ordering::SeqCst)
2032        );
2033        assert_eq!(store.list("data/").await.unwrap().len(), 2);
2034        let sequencer = Arc::new(
2035            ManifestSequencer::open(store.clone(), "manifest/")
2036                .await
2037                .unwrap(),
2038        );
2039        let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2040        assert!(
2041            reopened
2042                .fetch(&PartitionKey("p".into()), 0, 100)
2043                .await
2044                .unwrap()
2045                .is_empty()
2046        );
2047        assert_eq!(
2048            reopened
2049                .produce(
2050                    PartitionKey("p".into()),
2051                    Bytes::from_static(b"after-reopen"),
2052                    1,
2053                    (),
2054                    Durability::Sequenced
2055                )
2056                .await
2057                .unwrap()
2058                .base_offset,
2059            Some(0)
2060        );
2061        let rows = reopened
2062            .fetch(&PartitionKey("p".into()), 0, 100)
2063            .await
2064            .unwrap();
2065        assert_eq!(rows.len(), 1);
2066        assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2067        assert_eq!(store.list("data/").await.unwrap().len(), 3);
2068        tokio::task::spawn_blocking(move || drop(reopened))
2069            .await
2070            .unwrap();
2071    }
2072
2073    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2074    async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2075        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2076        let (release_tx, release_rx) = oneshot::channel();
2077        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2078            inner: MemoryBlobStore::new(),
2079            entered: entered_tx,
2080            release: Mutex::new(Some(release_rx)),
2081            fail: false,
2082            panic_on_commit: false,
2083        });
2084        let sequencer = Arc::new(
2085            ManifestSequencer::open(blob.clone(), "manifest/")
2086                .await
2087                .unwrap(),
2088        );
2089        let mut config = FlushConfig::default();
2090        config.max_batches = 1;
2091        config.max_inflight_flushes = 4;
2092        config.linger = Duration::ZERO;
2093        config.budget.enabled = false;
2094        let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2095        engine
2096            .produce(
2097                PartitionKey("p".into()),
2098                Bytes::from_static(b"first"),
2099                1,
2100                (),
2101                Durability::Buffered,
2102            )
2103            .await
2104            .unwrap();
2105        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2106        engine
2107            .produce(
2108                PartitionKey("p".into()),
2109                Bytes::from_static(b"second"),
2110                1,
2111                (),
2112                Durability::Buffered,
2113            )
2114            .await
2115            .unwrap();
2116        let shared = engine.shared.clone();
2117        let dropping = tokio::task::spawn_blocking(move || drop(engine));
2118        tokio::time::timeout(Duration::from_secs(5), async {
2119            while !shared.queue.lock().unwrap().shutdown {
2120                tokio::time::sleep(Duration::from_millis(1)).await;
2121            }
2122        })
2123        .await
2124        .unwrap();
2125        let premature = dropping.is_finished();
2126        release_tx.send(()).unwrap();
2127        tokio::time::timeout(Duration::from_secs(5), dropping)
2128            .await
2129            .unwrap()
2130            .unwrap();
2131        assert!(!premature, "shutdown returned before manifest durability");
2132        let reopened = Arc::new(
2133            ManifestSequencer::open(blob.clone(), "manifest/")
2134                .await
2135                .unwrap(),
2136        );
2137        let engine = LogEngine::new(blob, reopened, config, "data/");
2138        let rows = engine
2139            .fetch(&PartitionKey("p".into()), 0, 100)
2140            .await
2141            .unwrap();
2142        assert_eq!(
2143            rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2144            vec![b"first".as_slice(), b"second".as_slice()]
2145        );
2146    }
2147
2148    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2149    async fn panicked_committer_closes_admission_and_fails_flush() {
2150        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2151        let (release_tx, release_rx) = oneshot::channel();
2152        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2153            inner: MemoryBlobStore::new(),
2154            entered: entered_tx,
2155            release: Mutex::new(Some(release_rx)),
2156            fail: false,
2157            panic_on_commit: true,
2158        });
2159        let sequencer = Arc::new(
2160            ManifestSequencer::open(blob.clone(), "manifest/")
2161                .await
2162                .unwrap(),
2163        );
2164        let mut config = FlushConfig::default();
2165        config.max_batches = 1;
2166        config.max_inflight_flushes = 4;
2167        config.linger = Duration::ZERO;
2168        config.budget.enabled = false;
2169        let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2170        let first = {
2171            let engine = engine.clone();
2172            tokio::spawn(async move {
2173                engine
2174                    .produce(
2175                        PartitionKey("p".into()),
2176                        Bytes::from_static(b"first"),
2177                        1,
2178                        (),
2179                        Durability::Sequenced,
2180                    )
2181                    .await
2182            })
2183        };
2184        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2185        engine
2186            .produce(
2187                PartitionKey("p".into()),
2188                Bytes::from_static(b"later"),
2189                1,
2190                (),
2191                Durability::Buffered,
2192            )
2193            .await
2194            .unwrap();
2195        let flushing = {
2196            let engine = engine.clone();
2197            tokio::spawn(async move { engine.flush().await })
2198        };
2199        release_tx.send(()).unwrap();
2200        assert!(
2201            tokio::time::timeout(Duration::from_secs(5), first)
2202                .await
2203                .unwrap()
2204                .unwrap()
2205                .is_err()
2206        );
2207        assert!(
2208            tokio::time::timeout(Duration::from_secs(5), flushing)
2209                .await
2210                .unwrap()
2211                .unwrap()
2212                .is_err()
2213        );
2214        tokio::time::timeout(Duration::from_secs(5), async {
2215            while !engine.shared.queue.lock().unwrap().shutdown {
2216                tokio::time::sleep(Duration::from_millis(1)).await;
2217            }
2218        })
2219        .await
2220        .unwrap();
2221        assert!(
2222            engine
2223                .produce(
2224                    PartitionKey("p".into()),
2225                    Bytes::from_static(b"rejected"),
2226                    1,
2227                    (),
2228                    Durability::Buffered
2229                )
2230                .await
2231                .is_err()
2232        );
2233        assert!(engine.flush().await.is_err());
2234    }
2235    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2236    async fn committed_reads_continue_during_new_manifest_publication() {
2237        for fail in [false, true] {
2238            let memory = MemoryBlobStore::new();
2239            let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2240            let seed_sequencer = Arc::new(
2241                ManifestSequencer::open(seed_blob.clone(), "manifest/")
2242                    .await
2243                    .unwrap(),
2244            );
2245            let mut config = FlushConfig::default();
2246            config.max_batches = 1;
2247            config.max_inflight_flushes = 4;
2248            config.linger = Duration::ZERO;
2249            config.budget.enabled = false;
2250            let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2251            seed.produce(
2252                PartitionKey("p".into()),
2253                Bytes::from_static(b"committed"),
2254                1,
2255                (),
2256                Durability::Sequenced,
2257            )
2258            .await
2259            .unwrap();
2260            drop(seed);
2261            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2262            let (release_tx, release_rx) = oneshot::channel();
2263            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2264                inner: memory,
2265                entered: entered_tx,
2266                release: Mutex::new(Some(release_rx)),
2267                fail,
2268                panic_on_commit: false,
2269            });
2270            let sequencer = Arc::new(
2271                ManifestSequencer::open(blob.clone(), "manifest/")
2272                    .await
2273                    .unwrap(),
2274            );
2275            let engine = Arc::new(LogEngine::new(
2276                blob.clone(),
2277                sequencer.clone(),
2278                config,
2279                "data/",
2280            ));
2281            let producing = {
2282                let engine = engine.clone();
2283                tokio::spawn(async move {
2284                    engine
2285                        .produce(
2286                            PartitionKey("p".into()),
2287                            Bytes::from_static(b"new"),
2288                            1,
2289                            (),
2290                            Durability::Sequenced,
2291                        )
2292                        .await
2293                })
2294            };
2295            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2296            let mut reading = tokio::task::spawn_blocking(move || {
2297                let p = PartitionKey("p".into());
2298                (
2299                    sequencer.high_watermark(&p).unwrap(),
2300                    sequencer.lookup(&p, 0).unwrap(),
2301                    sequencer.snapshot(),
2302                )
2303            });
2304            let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2305            let early_ack = producing.is_finished();
2306            // Release even on regression failure so all worker threads can drain.
2307            release_tx.send(()).unwrap();
2308            let outcome = producing.await.unwrap();
2309            assert_eq!(outcome.is_err(), fail);
2310            let reads_progressed = observed.is_ok();
2311            let (watermark, entries, snapshot) = match observed {
2312                Ok(result) => result.unwrap(),
2313                Err(_) => reading.await.unwrap(),
2314            };
2315            assert!(!early_ack);
2316            assert!(
2317                reads_progressed,
2318                "durable manifest publication blocked committed index reads"
2319            );
2320            assert_eq!(watermark, 1);
2321            assert_eq!(entries.len(), 1);
2322            assert_eq!(snapshot.manifest_count, 1);
2323            let rows = engine
2324                .fetch(&PartitionKey("p".into()), 0, 100)
2325                .await
2326                .unwrap();
2327            assert_eq!(rows.len(), if fail { 1 } else { 2 });
2328            assert_eq!(rows[0].payload.as_ref(), b"committed");
2329            drop(engine);
2330            let reopened = Arc::new(
2331                ManifestSequencer::open(blob.clone(), "manifest/")
2332                    .await
2333                    .unwrap(),
2334            );
2335            let engine = LogEngine::new(blob, reopened, config, "data/");
2336            assert_eq!(
2337                engine
2338                    .fetch(&PartitionKey("p".into()), 0, 100)
2339                    .await
2340                    .unwrap()
2341                    .len(),
2342                if fail { 1 } else { 2 }
2343            );
2344        }
2345    }
2346    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2347    async fn manifest_mutations_remain_serialized_while_readers_progress() {
2348        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2349        let (release_tx, release_rx) = oneshot::channel();
2350        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2351            inner: MemoryBlobStore::new(),
2352            entered: entered_tx,
2353            release: Mutex::new(Some(release_rx)),
2354            fail: false,
2355            panic_on_commit: false,
2356        });
2357        let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2358        let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2359            tokio::task::spawn_blocking(move || {
2360                sequencer.commit(&[CommitBatch {
2361                    partition: PartitionKey("p".into()),
2362                    record_count: 1,
2363                    location: BatchLocation {
2364                        object_id: key.into(),
2365                        byte_start: 0,
2366                        byte_len: 1,
2367                    },
2368                    meta: &(),
2369                }])
2370            })
2371        };
2372        let first = commit(sequencer.clone(), "data/first");
2373        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2374        let second = commit(sequencer.clone(), "data/second");
2375        let truncating = {
2376            let sequencer = sequencer.clone();
2377            tokio::task::spawn_blocking(move || {
2378                sequencer.truncate_before(&PartitionKey("p".into()), 1)
2379            })
2380        };
2381        let reading = {
2382            let sequencer = sequencer.clone();
2383            tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2384        };
2385        let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2386        release_tx.send(()).unwrap();
2387        assert_eq!(
2388            observed.expect("committed reads blocked").unwrap().unwrap(),
2389            0
2390        );
2391        let first = tokio::time::timeout(Duration::from_secs(5), first)
2392            .await
2393            .unwrap()
2394            .unwrap()
2395            .unwrap();
2396        let second = tokio::time::timeout(Duration::from_secs(5), second)
2397            .await
2398            .unwrap()
2399            .unwrap()
2400            .unwrap();
2401        let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2402            .await
2403            .unwrap()
2404            .unwrap()
2405            .unwrap();
2406        assert_eq!(
2407            first,
2408            vec![CommitOutcome::Assigned {
2409                base_offset: 0,
2410                record_count: 1
2411            }]
2412        );
2413        assert_eq!(
2414            second,
2415            vec![CommitOutcome::Assigned {
2416                base_offset: 1,
2417                record_count: 1
2418            }]
2419        );
2420        assert_eq!(dropped, vec!["data/first".to_string()]);
2421        let p = PartitionKey("p".into());
2422        assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2423        assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2424        let entries = sequencer.lookup(&p, 0).unwrap();
2425        assert_eq!(entries.len(), 1);
2426        assert_eq!(entries[0].location.object_id, "data/second");
2427        assert_eq!(sequencer.snapshot().manifest_count, 2);
2428    }
2429}
2430
2431#[cfg(test)]
2432mod idle_batch_wait_tests {
2433    use super::*;
2434    use std::sync::mpsc;
2435
2436    fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2437        Arc::new(Shared {
2438            queue: Mutex::new(Queue {
2439                items: VecDeque::new(),
2440                bytes: 0,
2441                bytes_in_use: 0,
2442                shutdown: false,
2443                next_seq: 1,
2444                completed_through: 0,
2445                first_failed: None,
2446                force_flush: false,
2447                flush_waiters: Vec::new(),
2448                last_enqueue: None,
2449                oldest_enqueue: None,
2450            }),
2451            cv: Condvar::new(),
2452            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2453            budget: Mutex::new(BudgetRuntime::new(config.budget)),
2454            flush_config: config,
2455        })
2456    }
2457
2458    #[test]
2459    fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2460        for linger in [Duration::ZERO, Duration::from_millis(1)] {
2461            for shutdown in [false, true] {
2462                let config = FlushConfig {
2463                    linger,
2464                    ..FlushConfig::default()
2465                };
2466                let shared = empty_shared(config);
2467                let (started_tx, started_rx) = mpsc::channel();
2468                let (result_tx, result_rx) = mpsc::channel();
2469                let worker_shared = shared.clone();
2470                let worker = std::thread::spawn(move || {
2471                    started_tx.send(()).unwrap();
2472                    assert!(
2473                        result_tx
2474                            .send(take_batch(&worker_shared, config, true))
2475                            .is_ok()
2476                    );
2477                });
2478                started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2479                let early = result_rx.recv_timeout(Duration::from_millis(100));
2480                let returned_while_idle = early.is_ok();
2481                {
2482                    let mut q = shared.queue.lock().unwrap();
2483                    if shutdown {
2484                        q.shutdown = true;
2485                    } else {
2486                        let payload = Bytes::from_static(b"first-after-idle");
2487                        q.bytes = payload.len();
2488                        q.bytes_in_use = payload.len();
2489                        q.next_seq = 2;
2490                        q.last_enqueue = Some(Instant::now());
2491                        q.oldest_enqueue = q.last_enqueue;
2492                        q.force_flush = true;
2493                        q.items.push_back(Pending {
2494                            partition: PartitionKey("p".into()),
2495                            record_count: 1,
2496                            payload,
2497                            meta: (),
2498                            durability: Durability::Buffered,
2499                            responder: None,
2500                            seq: 1,
2501                        });
2502                    }
2503                    shared.cv.notify_all();
2504                }
2505                let result = early
2506                    .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2507                    .unwrap();
2508                worker.join().unwrap();
2509                assert!(
2510                    !returned_while_idle,
2511                    "idle worker polled with linger={linger:?}"
2512                );
2513                match result {
2514                    TakeBatch::Shutdown => assert!(shutdown),
2515                    TakeBatch::Batch(items) => {
2516                        assert!(!shutdown);
2517                        assert_eq!(items.len(), 1);
2518                        assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2519                        assert_eq!(items[0].seq, 1);
2520                    }
2521                    TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2522                }
2523            }
2524        }
2525    }
2526}