Skip to main content

object_log/
engine.rs

1//! The buffered, multiplexing log engine.
2
3use crate::budget::{
4    BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8    BlobStore, CommitBatch, CommitOutcome, ConditionalCommitOutcome, IndexEntry, ObjectLogError,
9    PartitionKey, Sequencer,
10};
11use bytes::Bytes;
12use std::collections::{HashSet, VecDeque};
13use std::sync::{Arc, Condvar, Mutex};
14use std::thread::JoinHandle;
15use std::time::{Duration, Instant};
16use tokio::sync::oneshot;
17use tokio::task::JoinHandle as TokioJoinHandle;
18
19const STORAGE_RETRY_ATTEMPTS: usize = 5;
20const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
21
22/// The durability point a [`LogEngine::produce`] call resolves at.
23#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub enum Durability {
25    /// Resolve as soon as the batch is buffered (fire-and-forget; may be lost on
26    /// crash before the flush). No offset is returned.
27    Buffered,
28    /// Resolve once the containing object is durably PUT (survives crash). No
29    /// offset yet — the commit has not run.
30    Durable,
31    /// Resolve once the batch is durably PUT **and** sequenced (has a stable
32    /// offset). This is the strong, no-loss level.
33    Sequenced,
34}
35
36/// Flush-trigger policy for the engine's group-commit buffer.
37#[derive(Clone, Copy, Debug)]
38pub struct FlushConfig {
39    /// Hard ceiling on one sealed object's payload size (memory / object-store
40    /// physics). **Not** the primary packing control: under normal load packing
41    /// is `ingest_rate × linger`. Keep this high enough that **linger binds**
42    /// before size (default **1 GiB**). Lower only for tight RAM or backend caps.
43    pub max_bytes: usize,
44    /// Flush once this many batches are buffered (secondary ceiling).
45    pub max_batches: usize,
46    /// **Maximum** time a batch may wait before a deadline flush (hard ceiling).
47    /// This is the latency↔throughput control surface: longer wait ⇒ more bytes
48    /// per seal ⇒ fewer durable ops/s. The budget controller may use a shorter
49    /// *effective* linger when media is idle (TD-004). `ZERO` = seal as soon as
50    /// any data is buffered (no co-buffer wait). Default `50ms`.
51    pub linger: Duration,
52    /// Max sealed objects PUT concurrently. Default **1** (single-flight bulk
53    /// path). Raise for parallel S3 PUTs. Also bounds ready objects per atomic
54    /// commit when the sequencer opts in to multi-object commits.
55    pub max_inflight_flushes: usize,
56    /// Max bytes in the mutable queue plus in-flight seals. Producers block when
57    /// exceeded. Default **2 GiB**.
58    pub max_buffered_bytes: usize,
59    /// Durable-ops budget controller (default on for Fjord).
60    pub budget: BudgetConfig,
61}
62
63impl Default for FlushConfig {
64    fn default() -> Self {
65        Self {
66            // Physics/safety ceiling so linger defines segment size under load.
67            max_bytes: 1024 * 1024 * 1024,
68            max_batches: 10_000,
69            linger: Duration::from_millis(50),
70            // 1 = single-flight seal (best Local bulk); raise for parallel S3 PUTs.
71            max_inflight_flushes: 1,
72            max_buffered_bytes: 2 * 1024 * 1024 * 1024,
73            budget: BudgetConfig::default(),
74        }
75    }
76}
77
78/// Outcome of a [`LogEngine::produce`] call.
79#[derive(Clone, Debug, PartialEq, Eq)]
80pub struct AppendOutcome {
81    /// First assigned offset, when resolved at [`Durability::Sequenced`].
82    pub base_offset: Option<i64>,
83    /// Last assigned offset, when resolved at [`Durability::Sequenced`].
84    pub last_offset: Option<i64>,
85    /// Whether the batch is durably stored.
86    pub durable: bool,
87    /// Whether the batch has been sequenced (has an offset).
88    pub sequenced: bool,
89}
90
91/// A batch read back by [`LogEngine::fetch`], with its assigned base offset.
92#[derive(Clone, Debug, PartialEq, Eq)]
93pub struct FetchedBatch {
94    /// First offset of the batch.
95    pub base_offset: i64,
96    /// Number of records in the batch.
97    pub record_count: i32,
98    /// The opaque batch payload.
99    pub payload: Bytes,
100}
101
102/// Snapshot of the engine's buffering envelope.
103#[derive(Clone, Debug, PartialEq, Eq)]
104pub struct BufferStats {
105    /// Bytes still waiting in the mutable accumulation queue.
106    pub queued_bytes: usize,
107    /// Bytes owned by queued plus sealed in-flight flush work.
108    pub bytes_in_use: usize,
109    /// Batches still waiting in the mutable accumulation queue.
110    pub queued_batches: usize,
111    /// Configured upper bound for `bytes_in_use`.
112    pub max_buffered_bytes: usize,
113}
114
115/// An embedding operation lifetime, retained through queued work and started
116/// upload completion, including copied upload buffers and failure draining.
117/// It must not depend on the caller waiting for its acknowledgement.
118pub type AppendOwner = Arc<dyn std::any::Any + Send + Sync>;
119
120type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
121
122fn retryable_storage_error(err: &ObjectLogError) -> bool {
123    matches!(err, ObjectLogError::StorageUnavailable(_))
124}
125
126async fn retry_delay(attempt: usize) {
127    let multiplier = 1u32 << attempt.min(5);
128    tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
129}
130
131async fn put_chunks_with_retries(
132    blob: &Arc<dyn BlobStore>,
133    key: &str,
134    chunks: Vec<Bytes>,
135) -> Result<(), ObjectLogError> {
136    // `Bytes::clone` is refcount-only; Local put_chunks streams without re-merge.
137    let mut attempt = 0usize;
138    loop {
139        match blob.put_chunks(key, chunks.clone()).await {
140            Ok(()) => return Ok(()),
141            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
142                retry_delay(attempt).await;
143                attempt += 1;
144            }
145            Err(err) => return Err(err),
146        }
147    }
148}
149
150pub(crate) enum RangeRead {
151    Legacy,
152    Exact,
153}
154pub(crate) async fn get_range_with_retries(
155    blob: &Arc<dyn BlobStore>,
156    key: &str,
157    range: std::ops::Range<u64>,
158    mode: RangeRead,
159) -> Result<Option<Bytes>, ObjectLogError> {
160    let mut attempt = 0usize;
161    loop {
162        let result = match mode {
163            RangeRead::Legacy => blob.get_range(key, range.clone()).await,
164            RangeRead::Exact => blob.get_range_exact(key, range.clone()).await,
165        };
166        match result {
167            Ok(bytes) => return Ok(bytes),
168            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
169                retry_delay(attempt).await;
170                attempt += 1;
171            }
172            Err(err) => return Err(err),
173        }
174    }
175}
176
177struct Pending<M> {
178    partition: PartitionKey,
179    record_count: i32,
180    payload: Bytes,
181    meta: M,
182    /// Fence epoch carried into [`CommitBatch::epoch`].
183    epoch: u64,
184    durability: Durability,
185    responder: Option<Responder>,
186    /// Monotonic enqueue id for [`LogEngine::flush`] barriers.
187    seq: u64,
188    owner: Option<AppendOwner>,
189    expected_head: Option<i64>,
190}
191
192struct FlushWork<M> {
193    batch: Vec<Pending<M>>,
194    locations: Vec<BatchLocation>,
195    responders: Vec<(Durability, Option<Responder>)>,
196    bytes: usize,
197    /// Highest enqueue seq included in this flush object.
198    max_seq: u64,
199    /// Durable data objects represented by this ordered commit group.
200    data_objects: usize,
201    put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
202    put_started: Instant,
203    put_result: Option<Result<Duration, ObjectLogError>>,
204}
205
206struct CommitJob {
207    task: TokioJoinHandle<usize>,
208    first_seq: u64,
209}
210
211enum TakeBatch<M> {
212    Batch(Vec<Pending<M>>),
213    Empty,
214    Shutdown,
215}
216
217type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
218
219struct Queue<M> {
220    items: VecDeque<Pending<M>>,
221    bytes: usize,
222    bytes_in_use: usize,
223    shutdown: bool,
224    /// Next seq to assign on enqueue.
225    next_seq: u64,
226    /// Highest settled enqueue sequence; failures are retained separately.
227    completed_through: u64,
228    /// Earliest failed enqueue prefix. A later success cannot make it durable.
229    first_failed: Option<(u64, ObjectLogError)>,
230    /// When true, take_batch flushes even if under size/linger.
231    force_flush: bool,
232    /// Waiters: (barrier_seq inclusive, responder).
233    flush_waiters: Vec<FlushWaiter>,
234    /// Last successful produce enqueue (for idle early-flush).
235    last_enqueue: Option<Instant>,
236    /// Enqueue time of the front item (linger deadline = oldest + linger).
237    oldest_enqueue: Option<Instant>,
238}
239
240struct Shared<M> {
241    queue: Mutex<Queue<M>>,
242    cv: Condvar,
243    max_buffered_bytes: usize,
244    /// Budget controller + inspectable counters (TD-004).
245    budget: Mutex<BudgetRuntime>,
246    /// Max linger (operator ceiling) and budget config copy for the flush loop.
247    flush_config: FlushConfig,
248}
249
250/// A buffered, multiplexing append-log engine over a [`BlobStore`], with
251/// sequencing delegated to a [`Sequencer`].
252///
253/// `produce` group-commits: many batches across many partitions are multiplexed
254/// into one object, PUT durably, then handed to the sequencer in a single call —
255/// so PUT count is decoupled from produce count. A single flush worker preserves
256/// per-[`PartitionKey`] arrival order and never splits a partition across
257/// concurrent commits.
258pub struct LogEngine<S: Sequencer> {
259    shared: Arc<Shared<S::Meta>>,
260    blob: Arc<dyn BlobStore>,
261    sequencer: Arc<S>,
262    /// Data-object key prefix (`<prefix><counter:020>`, or
263    /// `<prefix><writer>/<epoch>/<counter>` when a writer id is set).
264    key_prefix: String,
265    writer_id: String,
266    flush_thread: Option<JoinHandle<()>>,
267}
268
269impl<S> LogEngine<S>
270where
271    S: Sequencer + 'static,
272    S::Meta: Send + 'static,
273{
274    /// Create an engine over `blob` and `sequencer` with the given flush policy.
275    /// Objects are keyed `<key_prefix><counter>`; pick a prefix unique to this
276    /// engine instance if several share a store.
277    pub fn new(
278        blob: Arc<dyn BlobStore>,
279        sequencer: Arc<S>,
280        config: FlushConfig,
281        key_prefix: impl Into<String>,
282    ) -> Self {
283        Self::new_with_writer(blob, sequencer, config, key_prefix, "")
284    }
285
286    /// Like [`new`](Self::new), but data objects are
287    /// `<key_prefix><writer_id>/<epoch>/<counter>` so two writers cannot
288    /// overwrite the same key. `writer_id` must not contain `/` or `..`.
289    pub fn new_with_writer(
290        blob: Arc<dyn BlobStore>,
291        sequencer: Arc<S>,
292        config: FlushConfig,
293        key_prefix: impl Into<String>,
294        writer_id: impl Into<String>,
295    ) -> Self {
296        if let Err(msg) = config.budget.validate() {
297            panic!("invalid FlushConfig.budget: {msg}");
298        }
299        if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
300            eprintln!("object-log flush config: {config:?}");
301        }
302        let budget_rt = BudgetRuntime::new(config.budget);
303        let shared = Arc::new(Shared {
304            queue: Mutex::new(Queue {
305                items: VecDeque::new(),
306                bytes: 0,
307                bytes_in_use: 0,
308                shutdown: false,
309                next_seq: 1, // first enqueued batch gets seq 1
310                completed_through: 0,
311                first_failed: None,
312                force_flush: false,
313                flush_waiters: Vec::new(),
314                last_enqueue: None,
315                oldest_enqueue: None,
316            }),
317            cv: Condvar::new(),
318            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
319            budget: Mutex::new(budget_rt),
320            flush_config: config,
321        });
322        let key_prefix = key_prefix.into();
323        let writer_id = writer_id.into();
324        let flush_thread = {
325            let shared = Arc::clone(&shared);
326            let blob = Arc::clone(&blob);
327            let sequencer = Arc::clone(&sequencer);
328            let prefix = key_prefix.clone();
329            let writer_id = writer_id.clone();
330            std::thread::Builder::new()
331                .name("object-log-flush".into())
332                .spawn(move || flush_loop(shared, blob, sequencer, config, prefix, writer_id))
333                .expect("spawn flush thread")
334        };
335        Self {
336            writer_id,
337            shared,
338            blob,
339            sequencer,
340            key_prefix,
341            flush_thread: Some(flush_thread),
342        }
343    }
344
345    /// Prefix used for data objects written by this engine.
346    pub fn data_prefix(&self) -> &str {
347        &self.key_prefix
348    }
349
350    /// Borrow the engine's blob store (e.g. for orphan reaping or inspection).
351    pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
352        &self.blob
353    }
354
355    /// Borrow the engine's sequencer.
356    pub fn sequencer(&self) -> &Arc<S> {
357        &self.sequencer
358    }
359
360    /// Writer id used in data-object keys. Empty when constructed with [`new`](Self::new).
361    pub fn writer_id(&self) -> &str {
362        &self.writer_id
363    }
364
365    /// Buffer a batch and resolve at the requested [`Durability`].
366    ///
367    /// The batch's fence epoch is `0`. Use [`produce_at_epoch`](Self::produce_at_epoch)
368    /// when the partition index is fenced.
369    pub async fn produce(
370        &self,
371        partition: PartitionKey,
372        payload: Bytes,
373        record_count: i32,
374        meta: S::Meta,
375        durability: Durability,
376    ) -> Result<AppendOutcome, ObjectLogError> {
377        self.produce_at_epoch(partition, payload, record_count, meta, durability, 0)
378            .await
379    }
380
381    /// [`produce`](Self::produce) at a caller-supplied fence epoch.
382    pub async fn produce_at_epoch(
383        &self,
384        partition: PartitionKey,
385        payload: Bytes,
386        record_count: i32,
387        meta: S::Meta,
388        durability: Durability,
389        epoch: u64,
390    ) -> Result<AppendOutcome, ObjectLogError> {
391        self.produce_at_epoch_with_owner(
392            partition,
393            payload,
394            record_count,
395            meta,
396            durability,
397            epoch,
398            None,
399            None,
400        )
401        .await
402    }
403
404    /// Append with an operation lifetime independent of caller cancellation.
405    /// Acknowledgement is not a terminal-owner notification: use the owner's
406    /// final Drop/callback to observe release of all physical work.
407    #[allow(clippy::too_many_arguments)]
408    pub async fn produce_owned_at_epoch(
409        &self,
410        partition: PartitionKey,
411        payload: Bytes,
412        record_count: i32,
413        meta: S::Meta,
414        durability: Durability,
415        epoch: u64,
416        owner: AppendOwner,
417    ) -> Result<AppendOutcome, ObjectLogError> {
418        self.produce_at_epoch_with_owner(
419            partition,
420            payload,
421            record_count,
422            meta,
423            durability,
424            epoch,
425            Some(owner),
426            None,
427        )
428        .await
429    }
430
431    /// Owned compare-and-append; acknowledgement requires a sequencing decision.
432    #[allow(clippy::too_many_arguments)]
433    pub async fn produce_conditional_at_epoch(
434        &self,
435        partition: PartitionKey,
436        payload: Bytes,
437        record_count: i32,
438        meta: S::Meta,
439        epoch: u64,
440        expected_head: i64,
441        owner: AppendOwner,
442    ) -> Result<AppendOutcome, ObjectLogError> {
443        self.produce_at_epoch_with_owner(
444            partition,
445            payload,
446            record_count,
447            meta,
448            Durability::Sequenced,
449            epoch,
450            Some(owner),
451            Some(expected_head),
452        )
453        .await
454    }
455
456    #[allow(clippy::too_many_arguments)]
457    async fn produce_at_epoch_with_owner(
458        &self,
459        partition: PartitionKey,
460        payload: Bytes,
461        record_count: i32,
462        meta: S::Meta,
463        durability: Durability,
464        epoch: u64,
465        owner: Option<AppendOwner>,
466        expected_head: Option<i64>,
467    ) -> Result<AppendOutcome, ObjectLogError> {
468        if let Some(head) = expected_head {
469            if head < 0 {
470                return Err(ObjectLogError::InvalidBatch(
471                    "negative expected head".into(),
472                ));
473            }
474            if !self.sequencer.supports_conditional_commit() {
475                return Err(ObjectLogError::InvalidConfig(
476                    "conditional commit unsupported".into(),
477                ));
478            }
479        }
480        if record_count <= 0 {
481            return Err(ObjectLogError::InvalidBatch(
482                "record_count must be > 0".into(),
483            ));
484        }
485        if matches!(durability, Durability::Buffered) {
486            self.enqueue(Pending {
487                partition,
488                record_count,
489                payload,
490                meta,
491                epoch,
492                durability,
493                responder: None,
494                owner,
495                expected_head,
496                seq: 0, // filled in enqueue
497            })?;
498            return Ok(AppendOutcome {
499                base_offset: None,
500                last_offset: None,
501                durable: false,
502                sequenced: false,
503            });
504        }
505        // fail_closed: reserve predicted media ops before waiting on flush.
506        {
507            let mut budget = self.shared.budget.lock().expect("poisoned");
508            if budget.config.enabled
509                && budget.config.mode == BudgetMode::FailClosed
510                && !budget.reserve_for_fail_closed(Instant::now())
511            {
512                return Err(ObjectLogError::BudgetExceeded(
513                    "insufficient durable-ops budget to admit produce".into(),
514                ));
515            }
516        }
517        // budget_priority: wait briefly for tokens when empty.
518        if self.shared.flush_config.budget.enabled
519            && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
520        {
521            let timeout = self.shared.flush_config.budget.admission_timeout;
522            let deadline = Instant::now() + timeout;
523            loop {
524                let mut budget = self.shared.budget.lock().expect("poisoned");
525                if budget.can_admit_now(Instant::now()) {
526                    break;
527                }
528                drop(budget);
529                if Instant::now() >= deadline {
530                    return Err(ObjectLogError::BudgetExceeded(
531                        "timed out waiting for durable-ops budget".into(),
532                    ));
533                }
534                std::thread::sleep(Duration::from_millis(1));
535            }
536        }
537        let (tx, rx) = oneshot::channel();
538        self.enqueue(Pending {
539            partition,
540            record_count,
541            payload,
542            meta,
543            epoch,
544            durability,
545            responder: Some(tx),
546            owner,
547            expected_head,
548            seq: 0,
549        })?;
550        rx.await
551            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
552    }
553
554    /// Seal every batch enqueued **at or before** this call (barrier).
555    ///
556    /// Use after [`Durability::Buffered`] produces to wait for durable PUT (and
557    /// sequencing) of that prior work. Concurrent produces enqueued after this
558    /// call are not required to finish. An empty successful buffer returns immediately.
559    /// A failed PUT or commit remains an error for every barrier covering that
560    /// enqueue prefix, even after later appends succeed. Reopen the engine to
561    /// establish a new prefix from its successfully committed index.
562    pub async fn flush(&self) -> Result<(), ObjectLogError> {
563        let rx = {
564            let mut q = self.shared.queue.lock().expect("poisoned");
565            if q.shutdown {
566                return Err(ObjectLogError::Sequencer("engine shutting down".into()));
567            }
568            if q.next_seq == 1 {
569                // Nothing has ever been enqueued.
570                return Ok(());
571            }
572            let barrier = q.next_seq - 1;
573            if let Some((sequence, error)) = &q.first_failed
574                && *sequence <= barrier
575            {
576                return Err(error.clone());
577            }
578            if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
579                return Ok(());
580            }
581            let (tx, rx) = oneshot::channel();
582            q.force_flush = true;
583            q.flush_waiters.push((barrier, tx));
584            self.shared.cv.notify_all();
585            rx
586        };
587        rx.await
588            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
589    }
590
591    fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
592        let item_len = item.payload.len();
593        let max_buffered_bytes = self.shared.max_buffered_bytes;
594        let shared = Arc::clone(&self.shared);
595        let mut q = shared.queue.lock().expect("poisoned");
596        while !q.shutdown
597            && q.bytes_in_use > 0
598            && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
599        {
600            q = shared.cv.wait(q).expect("poisoned");
601        }
602        if q.shutdown {
603            return Err(ObjectLogError::Sequencer("engine shutting down".into()));
604        }
605        let now = Instant::now();
606        item.seq = q.next_seq;
607        q.next_seq = q.next_seq.saturating_add(1);
608        q.bytes += item_len;
609        q.bytes_in_use += item_len;
610        q.last_enqueue = Some(now);
611        if q.oldest_enqueue.is_none() {
612            q.oldest_enqueue = Some(now);
613        }
614        q.items.push_back(item);
615        shared.cv.notify_all();
616        Ok(())
617    }
618
619    /// Read batches covering offsets at/after `offset`, up to ~`max_bytes`.
620    pub async fn fetch(
621        &self,
622        partition: &PartitionKey,
623        offset: i64,
624        max_bytes: usize,
625    ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
626        let entries = self.sequencer.lookup(partition, offset)?;
627        let mut out = Vec::new();
628        let mut total = 0usize;
629        for e in entries {
630            if total >= max_bytes && !out.is_empty() {
631                break;
632            }
633            let batch = self.load_entry(&e).await?;
634            total += batch.payload.len();
635            out.push(batch);
636        }
637        Ok(out)
638    }
639
640    /// Stream batches from `offset` onward without materializing a full `Vec`.
641    ///
642    /// Calls `visit` once per index entry in order. Suitable for wide offset
643    /// windows (bounded-RAM replay). Stops and returns the error if `visit`
644    /// fails. Unlike [`Self::fetch`], there is no byte budget — visit every remaining
645    /// batch (or stop yourself inside `visit`).
646    pub async fn fetch_stream<F>(
647        &self,
648        partition: &PartitionKey,
649        offset: i64,
650        mut visit: F,
651    ) -> Result<(), ObjectLogError>
652    where
653        F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
654    {
655        let entries = self.sequencer.lookup(partition, offset)?;
656        for e in entries {
657            visit(self.load_entry(&e).await?)?;
658        }
659        Ok(())
660    }
661
662    async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
663        let start = e.location.byte_start as u64;
664        let end = start + e.location.byte_len as u64;
665        let bytes = get_range_with_retries(
666            &self.blob,
667            &e.location.object_id,
668            start..end,
669            RangeRead::Legacy,
670        )
671        .await?
672        .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
673        Ok(FetchedBatch {
674            base_offset: e.base_offset,
675            record_count: e.record_count,
676            payload: bytes,
677        })
678    }
679
680    /// Drop the partition's log below `offset` and delete any object that thereby
681    /// becomes fully unreferenced.
682    pub async fn truncate_before(
683        &self,
684        partition: &PartitionKey,
685        offset: i64,
686    ) -> Result<(), ObjectLogError> {
687        let dead = self.sequencer.truncate_before(partition, offset)?;
688        for object_id in dead {
689            self.blob.delete(&object_id).await?;
690        }
691        Ok(())
692    }
693
694    /// Delete data-prefix objects that are not in `live`.
695    ///
696    /// **Safety:** call only when this engine (and any other writer using the
697    /// same prefix) is quiescent — otherwise an in-flight put that has not yet
698    /// been committed may be deleted as an "orphan". Prefer running offline or
699    /// after drop. Does not touch keys outside [`data_prefix`](Self::data_prefix)
700    /// (e.g. a separate manifest prefix is safe).
701    ///
702    /// Returns the deleted object keys.
703    pub async fn reap_orphans(
704        &self,
705        live: &HashSet<String>,
706    ) -> Result<Vec<String>, ObjectLogError> {
707        reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
708    }
709
710    /// Return a point-in-time snapshot of queued and in-flight payload bytes.
711    pub fn buffer_stats(&self) -> BufferStats {
712        let q = self.shared.queue.lock().expect("poisoned");
713        BufferStats {
714            queued_bytes: q.bytes,
715            bytes_in_use: q.bytes_in_use,
716            queued_batches: q.items.len(),
717            max_buffered_bytes: self.shared.max_buffered_bytes,
718        }
719    }
720
721    /// Inspect budget controller layers and counters (TD-004).
722    pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
723        let now = Instant::now();
724        let mut budget = self.shared.budget.lock().expect("poisoned");
725        budget.refill(now);
726        let effective = budget.effective_budget_per_sec;
727        let reason = if !budget.config.enabled {
728            EffectiveReason::Disabled
729        } else if budget.config.budget_per_sec_cap.is_some()
730            && budget
731                .config
732                .budget_per_sec_cap
733                .is_some_and(|c| (effective - c).abs() < 1e-9)
734        {
735            EffectiveReason::ConfigCap
736        } else if budget.ongoing_capacity.is_some() {
737            EffectiveReason::Ongoing
738        } else if budget.startup_capacity.is_some() {
739            EffectiveReason::StartupProbe
740        } else {
741            EffectiveReason::DefaultCapacity
742        };
743        let (queued, last_enq) = {
744            let q = self.shared.queue.lock().expect("poisoned");
745            (q.bytes, q.last_enqueue)
746        };
747        let early = budget.allow_early_flush(now, queued, last_enq);
748        let max_linger = self.shared.flush_config.linger;
749        let eff_linger = if !budget.config.enabled {
750            max_linger
751        } else if early || max_linger.is_zero() {
752            Duration::ZERO
753        } else {
754            max_linger
755        };
756        PipelineSnapshot {
757            budget_per_sec: EffectiveKnob {
758                configured: budget.config.budget_per_sec_cap,
759                startup_measured: budget.startup_capacity,
760                ongoing_measured: budget.ongoing_capacity,
761                effective,
762                reason,
763            },
764            effective_linger_ms: EffectiveKnob {
765                configured: Some(max_linger.as_millis() as u64),
766                startup_measured: None,
767                ongoing_measured: None,
768                effective: eff_linger.as_millis() as u64,
769                reason: if early {
770                    EffectiveReason::Ongoing
771                } else {
772                    EffectiveReason::Configured
773                },
774            },
775            max_linger_ms: max_linger.as_millis() as u64,
776            token_fill_ratio: budget.fill_ratio(),
777            media_ops_total: budget.media_ops_total,
778            overdraft_total: budget.overdraft_total,
779            flushes_total: budget.flushes_total,
780            undersized_deadline_flushes: budget.undersized_deadline_flushes,
781            predicted_media_ops: budget.predicted_media_ops,
782            budget_enabled: budget.config.enabled,
783            budget_mode: budget.config.mode,
784        }
785    }
786}
787
788/// Delete objects under `data_prefix` whose keys are absent from `live`.
789///
790/// See [`LogEngine::reap_orphans`] for safety notes. `live` is typically
791/// [`InMemorySequencer::live_object_ids`](crate::InMemorySequencer::live_object_ids)
792/// or [`ManifestSequencer::live_object_ids`](crate::ManifestSequencer::live_object_ids).
793pub async fn reap_orphans(
794    blob: &dyn BlobStore,
795    data_prefix: &str,
796    live: &HashSet<String>,
797) -> Result<Vec<String>, ObjectLogError> {
798    let keys = blob.list(data_prefix).await?;
799    let mut deleted = Vec::new();
800    for key in keys {
801        if live.contains(&key) {
802            continue;
803        }
804        blob.delete(&key).await?;
805        deleted.push(key);
806    }
807    Ok(deleted)
808}
809
810impl<S: Sequencer> Drop for LogEngine<S> {
811    fn drop(&mut self) {
812        {
813            let mut q = self.shared.queue.lock().expect("poisoned");
814            q.shutdown = true;
815        }
816        self.shared.cv.notify_all();
817        if let Some(t) = self.flush_thread.take() {
818            let _ = t.join();
819        }
820    }
821}
822
823/// Highest numeric suffix already present under `data_prefix`.
824///
825/// Empty `writer_id` reads legacy keys `{prefix}{counter:020}`. A writer id
826/// reads `{prefix}{writer}/{epoch}/{counter}` and ignores other writers, so a
827/// restarted process with a new writer id cannot overwrite their objects.
828async fn recover_data_object_counter(
829    blob: &dyn BlobStore,
830    prefix: &str,
831    writer_id: &str,
832) -> Result<u64, ObjectLogError> {
833    let keys = blob.list(prefix).await?;
834    let mut max = 0u64;
835    let writer_prefix = format!("{prefix}{writer_id}/");
836    for key in keys {
837        let Some(suffix) = key.strip_prefix(prefix) else {
838            continue;
839        };
840        if writer_id.is_empty() {
841            if let Ok(n) = suffix.parse::<u64>() {
842                max = max.max(n);
843            }
844            continue;
845        }
846        let Some(rest) = key.strip_prefix(&writer_prefix) else {
847            continue;
848        };
849        if let Some(n) = rest
850            .rsplit('/')
851            .next()
852            .and_then(|tail| tail.parse::<u64>().ok())
853        {
854            max = max.max(n);
855        }
856    }
857    Ok(max)
858}
859
860fn data_object_key(prefix: &str, writer_id: &str, epoch: u64, counter: u64) -> String {
861    if writer_id.is_empty() {
862        format!("{prefix}{counter:020}")
863    } else {
864        format!("{prefix}{writer_id}/{epoch:020}/{counter:020}")
865    }
866}
867
868fn flush_loop<S>(
869    shared: Arc<Shared<S::Meta>>,
870    blob: Arc<dyn BlobStore>,
871    sequencer: Arc<S>,
872    config: FlushConfig,
873    prefix: String,
874    writer_id: String,
875) where
876    S: Sequencer + 'static,
877    S::Meta: Send + 'static,
878{
879    let max_inflight = config.max_inflight_flushes.max(1);
880    let commit_group_limit = if sequencer.supports_multi_object_commit() {
881        max_inflight
882    } else {
883        1
884    };
885    let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
886        .ok()
887        .and_then(|v| v.parse::<usize>().ok())
888        .filter(|v| *v > 0)
889        .unwrap_or_else(|| max_inflight.min(8));
890    let rt = tokio::runtime::Builder::new_multi_thread()
891        .worker_threads(worker_threads)
892        .enable_all()
893        .build()
894        .expect("flush runtime");
895    // Resume the data-object counter past any keys already under `prefix`. Restarting
896    // at 0 on reopen overwrites sealed objects while manifests still point at the old
897    // byte ranges → RangeOutOfBounds / mid-JSON EOF on fetch (fireweed-481d3e43).
898    let mut counter = match rt.block_on(recover_data_object_counter(
899        blob.as_ref(),
900        &prefix,
901        &writer_id,
902    )) {
903        Ok(counter) => counter,
904        Err(error) => {
905            // A failed listing is not an empty prefix. Fail admission before
906            // any PUT can reuse a sealed object's name.
907            let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
908            return;
909        }
910    };
911    let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
912    let mut active_puts = 0usize;
913    let mut committing: Option<CommitJob> = None;
914    let mut shutdown = false;
915
916    loop {
917        while !shutdown && active_puts < max_inflight {
918            // Poll the commit job even with no queued uploads: it owns bytes
919            // whose release may be needed to admit the next producer.
920            let wait_for_more = pending.is_empty() && committing.is_none();
921            match take_batch(&shared, config, wait_for_more) {
922                TakeBatch::Batch(batch) => {
923                    counter += 1;
924                    let concurrent = max_inflight > 1;
925                    let work = start_flush_work(
926                        &rt, &blob, &prefix, &writer_id, counter, batch, concurrent,
927                    );
928                    if concurrent {
929                        active_puts += 1;
930                    }
931                    pending.push_back(work);
932                }
933                TakeBatch::Empty => break,
934                TakeBatch::Shutdown => shutdown = true,
935            }
936        }
937
938        let mut made_progress = false;
939        for work in pending.iter_mut() {
940            let Some(put) = work.put.as_ref() else {
941                continue;
942            };
943            if !put.is_finished() {
944                continue;
945            }
946            let put = work.put.take().expect("put handle exists");
947            let elapsed = work.put_started.elapsed();
948            work.put_result = Some(match rt.block_on(put) {
949                Ok(Ok(())) => Ok(elapsed),
950                Ok(Err(e)) => Err(e),
951                Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
952                    "flush task failed: {e}"
953                ))),
954            });
955            active_puts = active_puts.saturating_sub(1);
956            made_progress = true;
957        }
958
959        if committing
960            .as_ref()
961            .is_some_and(|job| job.task.is_finished())
962        {
963            let job = committing.take().expect("completed commit exists");
964            match rt.block_on(job.task) {
965                Ok(released) => {
966                    let mut q = shared.queue.lock().expect("poisoned");
967                    q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
968                    shared.cv.notify_all();
969                }
970                Err(error) => {
971                    let uploads = fail_engine_and_take_pending_uploads(
972                        &shared,
973                        &mut pending,
974                        job.first_seq,
975                        ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
976                    );
977                    // A shared runtime can outlive this engine. Await accepted uploads
978                    // explicitly so close/reopen/reaping cannot race orphan writes.
979                    // They are never sequenced after the failed commit.
980                    for upload in uploads {
981                        let _ = rt.block_on(upload);
982                    }
983                    return;
984                }
985            }
986            made_progress = true;
987        }
988
989        if committing.is_none()
990            && let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit)
991        {
992            if max_inflight == 1 {
993                // Preserve the low-overhead single-flight path.
994                let released = finish_flush_work(&shared, &blob, &sequencer, work);
995                let mut q = shared.queue.lock().expect("poisoned");
996                q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
997                shared.cv.notify_all();
998            } else {
999                // One ordered committer; uploads can progress during its
1000                // durable manifest I/O. Bytes remain charged until completion.
1001                let first_seq = work
1002                    .batch
1003                    .iter()
1004                    .map(|p| p.seq)
1005                    .min()
1006                    .unwrap_or(work.max_seq);
1007                let shared = shared.clone();
1008                let blob = blob.clone();
1009                let sequencer = sequencer.clone();
1010                committing = Some(CommitJob {
1011                    task: rt.spawn_blocking(move || {
1012                        finish_flush_work(&shared, &blob, &sequencer, work)
1013                    }),
1014                    first_seq,
1015                });
1016            }
1017            made_progress = true;
1018        }
1019
1020        if pending.is_empty() && committing.is_none() {
1021            {
1022                let mut q = shared.queue.lock().expect("poisoned");
1023                if q.items.is_empty() && q.bytes_in_use == 0 {
1024                    // Nothing in flight: any remaining waiters with barrier already
1025                    // covered (or no work after last seal) can complete.
1026                    notify_flush_waiters(&mut q);
1027                }
1028            }
1029            shared.cv.notify_all();
1030            if shutdown {
1031                // Fail any remaining flush waiters.
1032                let mut q = shared.queue.lock().expect("poisoned");
1033                for (_, tx) in q.flush_waiters.drain(..) {
1034                    let _ = tx.send(Err(ObjectLogError::Sequencer(
1035                        "engine shutting down".into(),
1036                    )));
1037                }
1038                return;
1039            }
1040            continue;
1041        }
1042
1043        if !made_progress {
1044            std::thread::sleep(Duration::from_millis(1));
1045        }
1046    }
1047}
1048
1049/// Close admission and fail queued callers/barriers after an unrecoverable
1050/// startup or commit failure. Return started uploads for explicit draining.
1051fn fail_engine_and_take_pending_uploads<M>(
1052    shared: &Arc<Shared<M>>,
1053    pending: &mut VecDeque<FlushWork<M>>,
1054    first_seq: u64,
1055    error: ObjectLogError,
1056) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
1057    let mut uploads = Vec::new();
1058    for mut work in pending.drain(..) {
1059        if let Some(put) = work.put.take() {
1060            uploads.push(put);
1061        }
1062        send_storage_error(&mut work.responders, error.clone());
1063    }
1064    // Recover a poisoned guard only to close the engine, never to resume writes.
1065    let mut q = shared
1066        .queue
1067        .lock()
1068        .unwrap_or_else(|poisoned| poisoned.into_inner());
1069    q.shutdown = true;
1070    q.first_failed.get_or_insert((first_seq, error.clone()));
1071    let rejected = std::mem::take(&mut q.items);
1072    q.bytes = 0;
1073    // Keep outstanding bytes conservatively charged until the failed engine
1074    // and its accepted uploads have completed.
1075    for (_, tx) in q.flush_waiters.drain(..) {
1076        let _ = tx.send(Err(error.clone()));
1077    }
1078    q.force_flush = false;
1079    shared.cv.notify_all();
1080    drop(q);
1081    // Embedding lifetime owners may run terminal callbacks. Never drop those
1082    // (or admission-carrying payload owners) while holding the engine lock.
1083    for mut item in rejected {
1084        if let Some(tx) = item.responder.take() {
1085            let _ = tx.send(Err(error.clone()));
1086        }
1087    }
1088    uploads
1089}
1090
1091/// Combine only an already-ready contiguous success prefix. Never wait to fill
1092/// a group or cross an unfinished/failed PUT; preserve object and batch order.
1093fn take_ready_commit_group<M>(
1094    pending: &mut VecDeque<FlushWork<M>>,
1095    max_objects: usize,
1096) -> Option<FlushWork<M>> {
1097    pending.front()?.put_result.as_ref()?;
1098    let mut group = pending.pop_front().expect("ready front exists");
1099    if group.put_result.as_ref().is_some_and(Result::is_err) {
1100        return Some(group);
1101    }
1102    while group.data_objects < max_objects.max(1) {
1103        let Some(next) = pending.front() else { break };
1104        if !next.put_result.as_ref().is_some_and(Result::is_ok)
1105            || group.data_objects.saturating_add(next.data_objects) > max_objects
1106        {
1107            break;
1108        }
1109        let mut next = pending.pop_front().expect("ready successor exists");
1110        group.batch.append(&mut next.batch);
1111        group.locations.append(&mut next.locations);
1112        group.responders.append(&mut next.responders);
1113        group.bytes = group
1114            .bytes
1115            .checked_add(next.bytes)
1116            .expect("bounded pending bytes");
1117        group.max_seq = group.max_seq.max(next.max_seq);
1118        group.data_objects += next.data_objects;
1119        let next_elapsed = next.put_result.take().unwrap().unwrap();
1120        let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
1121        *elapsed = (*elapsed).max(next_elapsed);
1122    }
1123    Some(group)
1124}
1125
1126/// How long to sleep before re-evaluating a seal decision.
1127/// - `ZERO` means seal now (early-flush or linger deadline hit).
1128/// - Otherwise wait at most this long (interrupted by new produces).
1129fn effective_linger<M>(
1130    shared: &Shared<M>,
1131    config: FlushConfig,
1132    queued_bytes: usize,
1133    last_enqueue: Option<Instant>,
1134    oldest_enqueue: Option<Instant>,
1135) -> Duration {
1136    if config.linger.is_zero() {
1137        return Duration::ZERO;
1138    }
1139    let now = Instant::now();
1140    if config.budget.enabled {
1141        let mut budget = shared.budget.lock().expect("poisoned");
1142        budget.refill(now);
1143        if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
1144            budget.note_early_flush(now);
1145            return Duration::ZERO;
1146        }
1147    }
1148    // True linger: seal when the oldest buffered item reaches max wait.
1149    if let Some(oldest) = oldest_enqueue {
1150        let deadline = oldest + config.linger;
1151        if now >= deadline {
1152            return Duration::ZERO;
1153        }
1154        let until_deadline = deadline.saturating_duration_since(now);
1155        // Also wake when idle gate could open, so sparse produces don't wait full linger.
1156        if config.budget.enabled
1157            && !config.budget.early_flush_idle.is_zero()
1158            && let Some(last) = last_enqueue
1159        {
1160            let idle_at = last + config.budget.early_flush_idle;
1161            if idle_at > now && idle_at < deadline {
1162                return idle_at.saturating_duration_since(now);
1163            }
1164        }
1165        return until_deadline;
1166    }
1167    config.linger
1168}
1169
1170fn take_batch<M>(
1171    shared: &Arc<Shared<M>>,
1172    config: FlushConfig,
1173    wait_when_empty: bool,
1174) -> TakeBatch<M> {
1175    let mut q = shared.queue.lock().expect("poisoned");
1176    loop {
1177        if q.items.is_empty() {
1178            if q.shutdown {
1179                return TakeBatch::Shutdown;
1180            }
1181            if !wait_when_empty {
1182                return TakeBatch::Empty;
1183            }
1184            // No pending work needs a timer. Enqueue and shutdown change the
1185            // predicate under this mutex and notify. Waiting without
1186            // dropping/reacquiring it first also avoids a lost notification.
1187            q = shared.cv.wait(q).expect("poisoned");
1188            continue;
1189        }
1190
1191        let linger = {
1192            let queued = q.bytes;
1193            let last_enq = q.last_enqueue;
1194            let oldest = q.oldest_enqueue;
1195            drop(q);
1196            let l = effective_linger(shared, config, queued, last_enq, oldest);
1197            q = shared.queue.lock().expect("poisoned");
1198            l
1199        };
1200
1201        let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1202        let force = q.force_flush;
1203        let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1204        if triggered {
1205            if !size_trigger && !q.shutdown && !q.items.is_empty() {
1206                let mut budget = shared.budget.lock().expect("poisoned");
1207                if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1208                    budget.undersized_deadline_flushes += 1;
1209                }
1210            }
1211            break;
1212        }
1213
1214        // Short wait (until idle gate or linger deadline); re-evaluate, do not
1215        // seal solely because a probe sleep timed out.
1216        let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1217        q = guard;
1218    }
1219
1220    let force_drain = q.force_flush;
1221    let mut items = Vec::new();
1222    let mut bytes = 0usize;
1223    // On force_flush, still respect max_bytes so one object stays bounded, but
1224    // ignore max_batches so a barrier can drain large queues across objects.
1225    while let Some(item) = q.items.pop_front() {
1226        q.bytes = q.bytes.saturating_sub(item.payload.len());
1227        bytes += item.payload.len();
1228        items.push(item);
1229        if bytes >= config.max_bytes {
1230            break;
1231        }
1232        if !force_drain && items.len() >= config.max_batches {
1233            break;
1234        }
1235    }
1236    q.oldest_enqueue = if q.items.is_empty() {
1237        None
1238    } else {
1239        // Approximate: next seal window starts now for remaining items.
1240        Some(Instant::now())
1241    };
1242    // Keep forcing while a barrier still has queued work.
1243    if force_drain {
1244        q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1245    }
1246    if items.is_empty() {
1247        TakeBatch::Empty
1248    } else {
1249        TakeBatch::Batch(items)
1250    }
1251}
1252
1253fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1254    let done = q.completed_through;
1255    let mut i = 0;
1256    while i < q.flush_waiters.len() {
1257        let barrier = q.flush_waiters[i].0;
1258        let failure = q
1259            .first_failed
1260            .as_ref()
1261            .filter(|(sequence, _)| *sequence <= barrier)
1262            .map(|(_, error)| error.clone());
1263        if barrier <= done || failure.is_some() {
1264            let (_, tx) = q.flush_waiters.swap_remove(i);
1265            let _ = tx.send(failure.map_or(Ok(()), Err));
1266        } else {
1267            i += 1;
1268        }
1269    }
1270    if q.flush_waiters.is_empty() {
1271        q.force_flush = false;
1272    } else if q.items.is_empty() && q.bytes_in_use == 0 {
1273        // Barriers still waiting but no work — should not happen; clear force.
1274        q.force_flush = true;
1275    } else {
1276        q.force_flush = true;
1277    }
1278}
1279
1280fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1281    for (_, tx) in responders.iter_mut() {
1282        if let Some(tx) = tx.take() {
1283            let _ = tx.send(Err(err.clone()));
1284        }
1285    }
1286}
1287
1288fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1289    for (durability, tx) in responders.iter_mut() {
1290        if *durability == Durability::Durable
1291            && let Some(tx) = tx.take()
1292        {
1293            let _ = tx.send(Ok(AppendOutcome {
1294                base_offset: None,
1295                last_offset: None,
1296                durable: true,
1297                sequenced: false,
1298            }));
1299        }
1300    }
1301}
1302
1303fn prepare_flush_work<M>(
1304    prefix: &str,
1305    writer_id: &str,
1306    counter: u64,
1307    mut batch: Vec<Pending<M>>,
1308) -> (FlushWork<M>, String, Vec<Bytes>) {
1309    let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1310    let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1311    let epoch = batch.first().map(|pending| pending.epoch).unwrap_or(0);
1312    let key = data_object_key(prefix, writer_id, epoch, counter);
1313    let mut offset = 0usize;
1314    for p in &batch {
1315        let start = offset as u32;
1316        offset += p.payload.len();
1317        chunks.push(p.payload.clone());
1318        locations.push(BatchLocation {
1319            object_id: key.clone(),
1320            byte_start: start,
1321            byte_len: p.payload.len() as u32,
1322        });
1323    }
1324    if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1325        eprintln!(
1326            "object-log seal: key={key} batches={} bytes={offset}",
1327            batch.len()
1328        );
1329    }
1330    let responders: Vec<(Durability, Option<Responder>)> = batch
1331        .iter_mut()
1332        .map(|p| (p.durability, p.responder.take()))
1333        .collect();
1334    let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1335    let work = FlushWork {
1336        batch,
1337        locations,
1338        responders,
1339        bytes: offset,
1340        max_seq,
1341        data_objects: 1,
1342        put: None,
1343        put_started: Instant::now(),
1344        put_result: None,
1345    };
1346    (work, key, chunks)
1347}
1348
1349/// Start a durable put for a sealed batch.
1350///
1351/// - `concurrent == false` (default `max_inflight_flushes == 1`): `block_on` the
1352///   put on the flush thread; Local uses `block_in_place` (no spawn_blocking queue).
1353/// - `concurrent == true`: spawn put tasks for parallel S3-style throughput.
1354fn start_flush_work<M>(
1355    rt: &tokio::runtime::Runtime,
1356    blob: &Arc<dyn BlobStore>,
1357    prefix: &str,
1358    writer_id: &str,
1359    counter: u64,
1360    batch: Vec<Pending<M>>,
1361    concurrent: bool,
1362) -> FlushWork<M> {
1363    let (mut work, key, chunks) = prepare_flush_work(prefix, writer_id, counter, batch);
1364    let owners = work
1365        .batch
1366        .iter()
1367        .filter_map(|pending| pending.owner.clone())
1368        .collect();
1369    let upload = OwnedUpload {
1370        blob: Arc::clone(blob),
1371        key,
1372        chunks,
1373        _owners: owners,
1374    };
1375    let _ = blob.take_media_op_stats();
1376    work.put_started = Instant::now();
1377    if concurrent {
1378        work.put = Some(rt.spawn(upload.execute()));
1379    } else {
1380        let put_result = rt.block_on(upload.execute());
1381        let elapsed = work.put_started.elapsed();
1382        work.put_result = Some(match put_result {
1383            Ok(()) => Ok(elapsed),
1384            Err(e) => Err(e),
1385        });
1386    }
1387    work
1388}
1389
1390// Retain embedding owners independently of FlushWork, including after engine
1391// failure drops pending batches while separately started uploads are drained.
1392struct OwnedUpload {
1393    blob: Arc<dyn BlobStore>,
1394    key: String,
1395    chunks: Vec<Bytes>,
1396    _owners: Vec<AppendOwner>,
1397}
1398impl OwnedUpload {
1399    async fn execute(mut self) -> Result<(), ObjectLogError> {
1400        let chunks = std::mem::take(&mut self.chunks);
1401        let result = put_chunks_with_retries(&self.blob, &self.key, chunks).await;
1402        drop(self);
1403        result
1404    }
1405}
1406
1407enum CommitResults {
1408    Legacy(Vec<CommitOutcome>),
1409    Conditional(Vec<ConditionalCommitOutcome>),
1410}
1411impl CommitResults {
1412    fn len(&self) -> usize {
1413        match self {
1414            Self::Legacy(rows) => rows.len(),
1415            Self::Conditional(rows) => rows.len(),
1416        }
1417    }
1418    fn into_iter(self) -> CommitResultIter {
1419        match self {
1420            Self::Legacy(rows) => CommitResultIter::Legacy(rows.into_iter()),
1421            Self::Conditional(rows) => CommitResultIter::Conditional(rows.into_iter()),
1422        }
1423    }
1424}
1425enum CommitResultIter {
1426    Legacy(std::vec::IntoIter<CommitOutcome>),
1427    Conditional(std::vec::IntoIter<ConditionalCommitOutcome>),
1428}
1429impl Iterator for CommitResultIter {
1430    type Item = ConditionalCommitOutcome;
1431    fn next(&mut self) -> Option<Self::Item> {
1432        match self {
1433            Self::Legacy(rows) => rows.next().map(ConditionalCommitOutcome::Committed),
1434            Self::Conditional(rows) => rows.next(),
1435        }
1436    }
1437}
1438
1439fn finish_flush_work<S, M>(
1440    shared: &Arc<Shared<M>>,
1441    blob: &Arc<dyn BlobStore>,
1442    sequencer: &Arc<S>,
1443    mut work: FlushWork<S::Meta>,
1444) -> usize
1445where
1446    S: Sequencer + 'static,
1447    S::Meta: Send + 'static,
1448{
1449    let release_bytes = work.bytes;
1450    let first_seq = work
1451        .batch
1452        .iter()
1453        .map(|pending| pending.seq)
1454        .min()
1455        .unwrap_or(work.max_seq);
1456    // Durable-then-sequence: the object PUT may have overlapped later PUTs, but
1457    // sequencer commits are still completed in object creation order.
1458    // Grouped put_ms is the largest observed member PUT duration, not a sum.
1459    let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1460    let put_elapsed = match work.put_result.take().expect("put result is ready") {
1461        Ok(elapsed) => elapsed,
1462        Err(e) => {
1463            send_storage_error(&mut work.responders, e.clone());
1464            let mut q = shared.queue.lock().expect("poisoned");
1465            // Unblock flush waiters covering this seq with the storage error.
1466            let done = work.max_seq;
1467            if done >= q.completed_through {
1468                q.completed_through = done;
1469            }
1470            q.first_failed.get_or_insert((first_seq, e));
1471            notify_flush_waiters(&mut q);
1472            shared.cv.notify_all();
1473            return release_bytes;
1474        }
1475    };
1476
1477    // Media ops for the data object put.
1478    let mut media_ops = blob
1479        .take_media_op_stats()
1480        .map(|s| s.media_ops)
1481        .unwrap_or(work.data_objects as u64); // fallback: 1 per successful put
1482
1483    // Signal Durable-level waiters now (after PUT, before commit).
1484    send_durable_acks(&mut work.responders);
1485
1486    // Sequence the ready group atomically, preserving each object's locations.
1487    let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1488        .batch
1489        .iter()
1490        .zip(work.locations.iter())
1491        .map(|(p, loc)| CommitBatch {
1492            partition: p.partition.clone(),
1493            record_count: p.record_count,
1494            location: loc.clone(),
1495            meta: &p.meta,
1496            epoch: p.epoch,
1497        })
1498        .collect();
1499
1500    let commit_started = timing.then(Instant::now);
1501    // Clear stats again so sequencer durable work (e.g. ManifestSequencer put)
1502    // on the shared store is counted into the same budget.
1503    let _ = blob.take_media_op_stats();
1504    let mut commit_error = None;
1505    let committed = if work
1506        .batch
1507        .iter()
1508        .any(|pending| pending.expected_head.is_some())
1509    {
1510        let heads: Vec<_> = work
1511            .batch
1512            .iter()
1513            .map(|pending| pending.expected_head)
1514            .collect();
1515        sequencer
1516            .commit_conditionally(&commit_batches, &heads)
1517            .map(CommitResults::Conditional)
1518    } else {
1519        sequencer.commit(&commit_batches).map(CommitResults::Legacy)
1520    }
1521    .and_then(|outcomes| {
1522        if outcomes.len() != commit_batches.len() {
1523            return Err(ObjectLogError::Sequencer(
1524                "wrong commit outcome count".into(),
1525            ));
1526        }
1527        Ok(outcomes)
1528    });
1529    match committed {
1530        Ok(outcomes) => {
1531            if let Some(commit_started) = commit_started {
1532                eprintln!(
1533                    "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1534                    work.bytes,
1535                    commit_batches.len(),
1536                    put_elapsed.as_millis(),
1537                    commit_started.elapsed().as_millis(),
1538                    work.data_objects
1539                );
1540            }
1541            if let Some(s) = blob.take_media_op_stats() {
1542                media_ops = media_ops.saturating_add(s.media_ops);
1543            }
1544            for ((outcome, (_, tx)), pending) in outcomes
1545                .into_iter()
1546                .zip(work.responders.iter_mut())
1547                .zip(work.batch.iter())
1548            {
1549                if let Some(tx) = tx.take() {
1550                    let outcome = match outcome {
1551                        ConditionalCommitOutcome::Committed(outcome) => outcome,
1552                        ConditionalCommitOutcome::HeadRejected { expected, current } => {
1553                            let _ =
1554                                tx.send(Err(ObjectLogError::HeadRejected { expected, current }));
1555                            continue;
1556                        }
1557                        ConditionalCommitOutcome::Unknown(error) => {
1558                            let _ = tx.send(Err(ObjectLogError::StorageUnavailable(format!(
1559                                "conditional publication outcome unknown: {error}"
1560                            ))));
1561                            continue;
1562                        }
1563                    };
1564                    let (base, last) = match outcome {
1565                        CommitOutcome::Assigned {
1566                            base_offset,
1567                            record_count,
1568                        } => (
1569                            Some(base_offset),
1570                            Some(base_offset + record_count as i64 - 1),
1571                        ),
1572                        CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1573                        CommitOutcome::Rejected { reason } => {
1574                            // The sequencer owns what a rejection means.
1575                            let error = sequencer.rejection_error(
1576                                &pending.partition,
1577                                pending.epoch,
1578                                reason,
1579                            );
1580                            let _ = tx.send(Err(error));
1581                            continue;
1582                        }
1583                    };
1584                    let _ = tx.send(Ok(AppendOutcome {
1585                        base_offset: base,
1586                        last_offset: last,
1587                        durable: true,
1588                        sequenced: true,
1589                    }));
1590                }
1591            }
1592        }
1593        Err(e) => {
1594            if let Some(s) = blob.take_media_op_stats() {
1595                media_ops = media_ops.saturating_add(s.media_ops);
1596            }
1597            send_storage_error(&mut work.responders, e.clone());
1598            commit_error = Some(e);
1599        }
1600    }
1601
1602    {
1603        let mut budget = shared.budget.lock().expect("poisoned");
1604        budget.consume_after_flush(media_ops, Instant::now());
1605    }
1606    {
1607        let mut q = shared.queue.lock().expect("poisoned");
1608        if let Some(error) = commit_error {
1609            q.first_failed.get_or_insert((first_seq, error));
1610        }
1611        if work.max_seq >= q.completed_through {
1612            q.completed_through = work.max_seq;
1613        }
1614        notify_flush_waiters(&mut q);
1615    }
1616    shared.cv.notify_all();
1617
1618    release_bytes
1619}
1620
1621#[cfg(test)]
1622mod flush_barrier_tests {
1623    use super::*;
1624
1625    #[test]
1626    fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1627        let error = ObjectLogError::Sequencer("manifest failed".into());
1628        let mut queue = Queue::<()> {
1629            items: VecDeque::new(),
1630            bytes: 0,
1631            bytes_in_use: 1,
1632            shutdown: false,
1633            next_seq: 4,
1634            completed_through: 2,
1635            first_failed: Some((2, error.clone())),
1636            force_flush: true,
1637            flush_waiters: Vec::new(),
1638            last_enqueue: None,
1639            oldest_enqueue: None,
1640        };
1641        let mut receivers = Vec::new();
1642        for barrier in [1, 2, 3] {
1643            let (tx, rx) = oneshot::channel();
1644            queue.flush_waiters.push((barrier, tx));
1645            receivers.push(rx);
1646        }
1647        notify_flush_waiters(&mut queue);
1648        assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1649        assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1650        assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1651        assert!(queue.flush_waiters.is_empty());
1652    }
1653}
1654
1655#[cfg(test)]
1656mod ready_commit_group_tests {
1657    use super::*;
1658    use crate::{ManifestSequencer, MemoryBlobStore};
1659    use async_trait::async_trait;
1660    use std::ops::Range;
1661
1662    type WorkFixture = (
1663        FlushWork<()>,
1664        String,
1665        Vec<Bytes>,
1666        oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1667    );
1668    fn work(n: u64, durability: Durability) -> WorkFixture {
1669        let (tx, rx) = oneshot::channel();
1670        let (mut work, key, chunks) = prepare_flush_work(
1671            "data/",
1672            "",
1673            n,
1674            vec![Pending {
1675                partition: PartitionKey("shared".into()),
1676                record_count: 1,
1677                payload: Bytes::from(format!("body-{n}")),
1678                meta: (),
1679                epoch: 0,
1680                durability,
1681                responder: Some(tx),
1682                seq: n,
1683                owner: None,
1684                expected_head: None,
1685            }],
1686        );
1687        work.put_result = Some(Ok(Duration::from_millis(n)));
1688        (work, key, chunks, rx)
1689    }
1690
1691    #[test]
1692    fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1693        let mut pending = (1..=5)
1694            .map(|n| work(n, Durability::Sequenced).0)
1695            .collect::<VecDeque<_>>();
1696        let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1697        let mut released = 0;
1698        for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1699            let group = take_ready_commit_group(&mut pending, 2).unwrap();
1700            assert_eq!(
1701                group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1702                expected
1703            );
1704            assert_eq!(group.data_objects, expected.len());
1705            assert_eq!(group.responders.len(), expected.len());
1706            assert_eq!(group.max_seq, *expected.last().unwrap());
1707            for (location, n) in group.locations.iter().zip(&expected) {
1708                assert_eq!(location.object_id, format!("data/{n:020}"));
1709                assert_eq!(location.byte_start, 0);
1710                assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1711            }
1712            released += group.bytes;
1713        }
1714        assert!(pending.is_empty());
1715        assert_eq!(released, original_bytes);
1716
1717        let mut first = work(1, Durability::Sequenced).0;
1718        first.put_result = None;
1719        let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1720        assert!(take_ready_commit_group(&mut pending, 8).is_none());
1721        assert_eq!(
1722            pending.len(),
1723            2,
1724            "a later ready PUT cannot pass an unfinished head"
1725        );
1726        pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1727        let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1728        assert_eq!(failed.data_objects, 1);
1729        assert!(failed.put_result.unwrap().is_err());
1730        assert_eq!(pending[0].max_seq, 2);
1731
1732        let mut failed = work(2, Durability::Sequenced).0;
1733        failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1734        let mut pending = VecDeque::from([
1735            work(1, Durability::Sequenced).0,
1736            failed,
1737            work(3, Durability::Sequenced).0,
1738        ]);
1739        assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1740        assert_eq!(
1741            pending.len(),
1742            2,
1743            "a success group cannot cross a failed PUT"
1744        );
1745        let mut pending = VecDeque::from([
1746            work(1, Durability::Sequenced).0,
1747            work(2, Durability::Sequenced).0,
1748        ]);
1749        assert_eq!(
1750            take_ready_commit_group(&mut pending, 1)
1751                .unwrap()
1752                .data_objects,
1753            1
1754        );
1755        assert_eq!(
1756            pending.len(),
1757            1,
1758            "one-object sequencers retain their call boundary"
1759        );
1760    }
1761
1762    struct GatedManifestStore {
1763        inner: MemoryBlobStore,
1764        entered: std::sync::mpsc::SyncSender<()>,
1765        release: Mutex<Option<oneshot::Receiver<()>>>,
1766        fail: bool,
1767        panic_on_commit: bool,
1768    }
1769
1770    #[async_trait]
1771    impl BlobStore for GatedManifestStore {
1772        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1773            if key.starts_with("manifest/") {
1774                let release = self.release.lock().unwrap().take();
1775                if let Some(release) = release {
1776                    self.entered.send(()).unwrap();
1777                    release.await.unwrap();
1778                    assert!(!self.panic_on_commit, "injected commit panic");
1779                    if self.fail {
1780                        return Err(ObjectLogError::StorageUnavailable(
1781                            "manifest failure".into(),
1782                        ));
1783                    }
1784                }
1785            }
1786            self.inner.put(key, value).await
1787        }
1788        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1789            self.inner.get(key).await
1790        }
1791        async fn get_range(
1792            &self,
1793            key: &str,
1794            range: Range<u64>,
1795        ) -> Result<Option<Bytes>, ObjectLogError> {
1796            self.inner.get_range(key, range).await
1797        }
1798        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1799            self.inner.list(prefix).await
1800        }
1801        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1802            self.inner.delete(key).await
1803        }
1804        fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1805            self.inner.take_media_op_stats()
1806        }
1807    }
1808
1809    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1810    async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1811        for fail in [false, true] {
1812            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1813            let (release_tx, release_rx) = oneshot::channel();
1814            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1815                inner: MemoryBlobStore::new(),
1816                entered: entered_tx,
1817                release: Mutex::new(Some(release_rx)),
1818                fail,
1819                panic_on_commit: false,
1820            });
1821            let sequencer = Arc::new(
1822                ManifestSequencer::open(blob.clone(), "manifest/")
1823                    .await
1824                    .unwrap(),
1825            );
1826            assert!(sequencer.supports_multi_object_commit());
1827            let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1828            let (mut second, second_key, second_chunks, mut durable_rx) =
1829                work(2, Durability::Durable);
1830            second.batch[0].partition = PartitionKey("other".into());
1831            let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1832            blob.put_chunks(&first_key, first_chunks).await.unwrap();
1833            blob.put_chunks(&second_key, second_chunks).await.unwrap();
1834            blob.put_chunks(&third_key, third_chunks).await.unwrap();
1835            let mut pending = VecDeque::from([first, second, third]);
1836            let group = take_ready_commit_group(&mut pending, 8).unwrap();
1837            let bytes = group.bytes;
1838            assert_eq!(group.data_objects, 3);
1839            let (flush_tx, mut flush_rx) = oneshot::channel();
1840            let config = FlushConfig {
1841                budget: BudgetConfig {
1842                    enabled: false,
1843                    ..BudgetConfig::default()
1844                },
1845                ..FlushConfig::default()
1846            };
1847            let shared: Arc<Shared<()>> = Arc::new(Shared {
1848                queue: Mutex::new(Queue {
1849                    items: VecDeque::new(),
1850                    bytes: 0,
1851                    bytes_in_use: bytes,
1852                    shutdown: false,
1853                    next_seq: 4,
1854                    completed_through: 0,
1855                    first_failed: None,
1856                    force_flush: true,
1857                    flush_waiters: vec![(3, flush_tx)],
1858                    last_enqueue: None,
1859                    oldest_enqueue: None,
1860                }),
1861                cv: Condvar::new(),
1862                max_buffered_bytes: bytes,
1863                budget: Mutex::new(BudgetRuntime::new(config.budget)),
1864                flush_config: config,
1865            });
1866            let worker = {
1867                let shared = shared.clone();
1868                let blob = blob.clone();
1869                let sequencer = sequencer.clone();
1870                std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1871            };
1872            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1873            assert!(matches!(
1874                sequenced_rx.try_recv(),
1875                Err(oneshot::error::TryRecvError::Empty)
1876            ));
1877            assert!(matches!(
1878                third_rx.try_recv(),
1879                Err(oneshot::error::TryRecvError::Empty)
1880            ));
1881            assert!(matches!(
1882                flush_rx.try_recv(),
1883                Err(oneshot::error::TryRecvError::Empty)
1884            ));
1885            let durable = durable_rx.try_recv().unwrap().unwrap();
1886            assert!(durable.durable && !durable.sequenced);
1887            release_tx.send(()).unwrap();
1888            assert_eq!(worker.join().unwrap(), bytes);
1889            if fail {
1890                assert!(sequenced_rx.await.unwrap().is_err());
1891                assert!(third_rx.await.unwrap().is_err());
1892                assert!(flush_rx.await.unwrap().is_err());
1893                assert_eq!(
1894                    sequencer
1895                        .high_watermark(&PartitionKey("shared".into()))
1896                        .unwrap(),
1897                    0
1898                );
1899                assert!(blob.list("manifest/").await.unwrap().is_empty());
1900            } else {
1901                assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1902                assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1903                flush_rx.await.unwrap().unwrap();
1904                assert_eq!(blob.list("manifest/").await.unwrap().len(), 3);
1905            }
1906            assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1907            let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1908                .await
1909                .unwrap();
1910            for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1911                let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1912                assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1913                for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1914                    assert_eq!(entry.base_offset, offset as i64);
1915                    let loc = &entry.location;
1916                    assert_eq!(loc.object_id, format!("data/{object:020}"));
1917                    let body = blob
1918                        .get_range(
1919                            &loc.object_id,
1920                            loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1921                        )
1922                        .await
1923                        .unwrap()
1924                        .unwrap();
1925                    assert_eq!(body, format!("body-{object}"));
1926                }
1927            }
1928        }
1929    }
1930
1931    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1932    async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1933        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1934        let (release_tx, release_rx) = oneshot::channel();
1935        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1936            inner: MemoryBlobStore::new(),
1937            entered: entered_tx,
1938            release: Mutex::new(Some(release_rx)),
1939            fail: false,
1940            panic_on_commit: false,
1941        });
1942        let sequencer = Arc::new(
1943            ManifestSequencer::open(blob.clone(), "manifest/")
1944                .await
1945                .unwrap(),
1946        );
1947        let config = FlushConfig {
1948            max_batches: 1,
1949            max_inflight_flushes: 4,
1950            linger: Duration::ZERO,
1951            budget: BudgetConfig {
1952                enabled: false,
1953                ..BudgetConfig::default()
1954            },
1955            ..FlushConfig::default()
1956        };
1957        let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1958        let first = {
1959            let engine = engine.clone();
1960            tokio::spawn(async move {
1961                engine
1962                    .produce(
1963                        PartitionKey("p".into()),
1964                        Bytes::from_static(b"first"),
1965                        1,
1966                        (),
1967                        Durability::Sequenced,
1968                    )
1969                    .await
1970            })
1971        };
1972        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1973        for n in 1..4 {
1974            engine
1975                .produce(
1976                    PartitionKey("p".into()),
1977                    Bytes::from(format!("later-{n}")),
1978                    1,
1979                    (),
1980                    Durability::Buffered,
1981                )
1982                .await
1983                .unwrap();
1984        }
1985        let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1986            loop {
1987                if blob.list("data/").await.unwrap().len() == 4 {
1988                    break;
1989                }
1990                tokio::time::sleep(Duration::from_millis(1)).await;
1991            }
1992        })
1993        .await
1994        .is_ok();
1995        let early_ack = first.is_finished();
1996        // Always release before assertions so a failed regression can drain.
1997        release_tx.send(()).unwrap();
1998        first.await.unwrap().unwrap();
1999        engine.flush().await.unwrap();
2000        let rows = engine
2001            .fetch(&PartitionKey("p".into()), 0, 1024)
2002            .await
2003            .unwrap();
2004        assert_eq!(
2005            rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
2006            vec![0, 1, 2, 3]
2007        );
2008        drop(engine);
2009        assert!(
2010            !early_ack,
2011            "sequenced acknowledgement preceded manifest durability"
2012        );
2013        assert!(
2014            upload_progress,
2015            "blocked manifest prevented independent data uploads"
2016        );
2017    }
2018
2019    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2020    async fn commit_completion_releases_admission_bytes_without_more_uploads() {
2021        let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
2022        let sequencer = Arc::new(
2023            ManifestSequencer::open(blob.clone(), "manifest/")
2024                .await
2025                .unwrap(),
2026        );
2027        let config = FlushConfig {
2028            max_bytes: 4,
2029            max_buffered_bytes: 4,
2030            max_batches: 1,
2031            max_inflight_flushes: 4,
2032            linger: Duration::ZERO,
2033            budget: BudgetConfig {
2034                enabled: false,
2035                ..BudgetConfig::default()
2036            },
2037        };
2038        let engine = LogEngine::new(blob, sequencer, config, "data/");
2039        engine
2040            .produce(
2041                PartitionKey("p".into()),
2042                Bytes::from_static(b"full"),
2043                1,
2044                (),
2045                Durability::Sequenced,
2046            )
2047            .await
2048            .unwrap();
2049        tokio::time::timeout(Duration::from_secs(5), async {
2050            while engine.buffer_stats().bytes_in_use != 0 {
2051                tokio::time::sleep(Duration::from_millis(1)).await;
2052            }
2053        })
2054        .await
2055        .expect("completed committer stranded admission bytes");
2056        engine
2057            .produce(
2058                PartitionKey("p".into()),
2059                Bytes::from_static(b"next"),
2060                1,
2061                (),
2062                Durability::Sequenced,
2063            )
2064            .await
2065            .unwrap();
2066        engine.flush().await.unwrap();
2067        assert_eq!(
2068            engine
2069                .fetch(&PartitionKey("p".into()), 0, 100)
2070                .await
2071                .unwrap()
2072                .len(),
2073            2
2074        );
2075    }
2076
2077    struct FailureInjectingStore {
2078        inner: Arc<dyn BlobStore>,
2079        second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
2080        second_upload_completed: std::sync::atomic::AtomicBool,
2081        fail_data_listing: std::sync::atomic::AtomicBool,
2082    }
2083
2084    impl FailureInjectingStore {
2085        fn new(inner: Arc<dyn BlobStore>) -> Self {
2086            Self {
2087                inner,
2088                second_upload_gate: Mutex::new(None),
2089                second_upload_completed: std::sync::atomic::AtomicBool::new(false),
2090                fail_data_listing: std::sync::atomic::AtomicBool::new(false),
2091            }
2092        }
2093    }
2094
2095    #[async_trait]
2096    impl BlobStore for FailureInjectingStore {
2097        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
2098            let gate = if key == "data/00000000000000000002" {
2099                self.second_upload_gate.lock().unwrap().take()
2100            } else {
2101                None
2102            };
2103            if let Some((entered, release)) = gate {
2104                entered.send(()).unwrap();
2105                release.await.unwrap();
2106                let result = self.inner.put(key, value).await;
2107                if result.is_ok() {
2108                    self.second_upload_completed
2109                        .store(true, std::sync::atomic::Ordering::SeqCst);
2110                }
2111                return result;
2112            }
2113            self.inner.put(key, value).await
2114        }
2115        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
2116            self.inner.get(key).await
2117        }
2118        async fn get_range(
2119            &self,
2120            key: &str,
2121            range: Range<u64>,
2122        ) -> Result<Option<Bytes>, ObjectLogError> {
2123            self.inner.get_range(key, range).await
2124        }
2125        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
2126            if prefix == "data/"
2127                && self
2128                    .fail_data_listing
2129                    .load(std::sync::atomic::Ordering::SeqCst)
2130            {
2131                return Err(ObjectLogError::StorageUnavailable(
2132                    "injected object listing failure".into(),
2133                ));
2134            }
2135            self.inner.list(prefix).await
2136        }
2137        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
2138            self.inner.delete(key).await
2139        }
2140    }
2141
2142    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2143    async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
2144        let inner = Arc::new(MemoryBlobStore::new());
2145        let key = "data/00000000000000000001";
2146        let original = Bytes::from_static(b"existing immutable object");
2147        inner.put(key, original.clone()).await.unwrap();
2148        let store = Arc::new(FailureInjectingStore::new(inner.clone()));
2149        store
2150            .fail_data_listing
2151            .store(true, std::sync::atomic::Ordering::SeqCst);
2152        let config = FlushConfig {
2153            max_batches: 1,
2154            max_inflight_flushes: 4,
2155            linger: Duration::ZERO,
2156            budget: BudgetConfig {
2157                enabled: false,
2158                ..BudgetConfig::default()
2159            },
2160            ..FlushConfig::default()
2161        };
2162        let engine = LogEngine::new(
2163            store,
2164            Arc::new(crate::InMemorySequencer::new()),
2165            config,
2166            "data/",
2167        );
2168        let result = tokio::time::timeout(
2169            Duration::from_secs(5),
2170            engine.produce(
2171                PartitionKey("p".into()),
2172                Bytes::from_static(b"must not overwrite"),
2173                1,
2174                (),
2175                Durability::Sequenced,
2176            ),
2177        )
2178        .await;
2179        let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
2180        tokio::task::spawn_blocking(move || drop(engine))
2181            .await
2182            .unwrap();
2183        let retained = inner.get(key).await.unwrap();
2184        assert_eq!(
2185            retained,
2186            Some(original),
2187            "failed listing must preserve sealed bytes"
2188        );
2189        assert!(
2190            result.unwrap().is_err(),
2191            "failed prefix recovery must reject admission"
2192        );
2193        assert!(
2194            flushed.unwrap().is_err(),
2195            "flush must report failed initialization"
2196        );
2197        assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
2198    }
2199
2200    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2201    async fn engines_commit_and_close_independently() {
2202        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2203        let (release_tx, release_rx) = oneshot::channel();
2204        let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2205            inner: MemoryBlobStore::new(),
2206            entered: entered_tx,
2207            release: Mutex::new(Some(release_rx)),
2208            fail: false,
2209            panic_on_commit: false,
2210        })));
2211        let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
2212        let config = FlushConfig {
2213            max_batches: 1,
2214            max_inflight_flushes: 4,
2215            linger: Duration::ZERO,
2216            budget: BudgetConfig {
2217                enabled: false,
2218                ..BudgetConfig::default()
2219            },
2220            ..FlushConfig::default()
2221        };
2222        let a_seq = Arc::new(
2223            ManifestSequencer::open(a_blob.clone(), "manifest/")
2224                .await
2225                .unwrap(),
2226        );
2227        let b_seq = Arc::new(
2228            ManifestSequencer::open(b_blob.clone(), "manifest/")
2229                .await
2230                .unwrap(),
2231        );
2232        let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
2233        a.produce(
2234            PartitionKey("p".into()),
2235            Bytes::from_static(b"a-first"),
2236            1,
2237            (),
2238            Durability::Buffered,
2239        )
2240        .await
2241        .unwrap();
2242        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2243        let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
2244        let b_result = tokio::time::timeout(
2245            Duration::from_secs(5),
2246            b.produce(
2247                PartitionKey("p".into()),
2248                Bytes::from_static(b"b-first"),
2249                1,
2250                (),
2251                Durability::Sequenced,
2252            ),
2253        )
2254        .await;
2255        let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
2256        let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
2257        // Always release the held manifest before assertions, including the pre-change red run.
2258        release_tx.send(()).unwrap();
2259        if b_closed.is_err() {
2260            closing_b.await.unwrap();
2261        }
2262        assert!(
2263            b_closed.is_ok(),
2264            "closing B waited for A's blocked manifest"
2265        );
2266        assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
2267        a.flush().await.unwrap();
2268        assert_eq!(
2269            a.produce(
2270                PartitionKey("p".into()),
2271                Bytes::from_static(b"a-second"),
2272                1,
2273                (),
2274                Durability::Sequenced
2275            )
2276            .await
2277            .unwrap()
2278            .base_offset,
2279            Some(1)
2280        );
2281        let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
2282        assert_eq!(
2283            rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
2284            vec![
2285                Bytes::from_static(b"a-first"),
2286                Bytes::from_static(b"a-second")
2287            ]
2288        );
2289        tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
2290    }
2291
2292    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2293    async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
2294        let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
2295        let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
2296        let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
2297        let (upload_release_tx, upload_release_rx) = oneshot::channel();
2298        let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2299            inner: MemoryBlobStore::new(),
2300            entered: manifest_entered_tx,
2301            release: Mutex::new(Some(manifest_release_rx)),
2302            fail: false,
2303            panic_on_commit: true,
2304        })));
2305        *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
2306        let config = FlushConfig {
2307            max_batches: 1,
2308            max_inflight_flushes: 4,
2309            linger: Duration::ZERO,
2310            budget: BudgetConfig {
2311                enabled: false,
2312                ..BudgetConfig::default()
2313            },
2314            ..FlushConfig::default()
2315        };
2316        let sequencer = Arc::new(
2317            ManifestSequencer::open(store.clone(), "manifest/")
2318                .await
2319                .unwrap(),
2320        );
2321        let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
2322        engine
2323            .produce(
2324                PartitionKey("p".into()),
2325                Bytes::from_static(b"first"),
2326                1,
2327                (),
2328                Durability::Buffered,
2329            )
2330            .await
2331            .unwrap();
2332        manifest_entered_rx
2333            .recv_timeout(Duration::from_secs(5))
2334            .unwrap();
2335        engine
2336            .produce(
2337                PartitionKey("p".into()),
2338                Bytes::from_static(b"second"),
2339                1,
2340                (),
2341                Durability::Buffered,
2342            )
2343            .await
2344            .unwrap();
2345        upload_entered_rx
2346            .recv_timeout(Duration::from_secs(5))
2347            .unwrap();
2348        manifest_release_tx.send(()).unwrap();
2349        tokio::time::timeout(Duration::from_secs(5), async {
2350            while !engine.shared.queue.lock().unwrap().shutdown {
2351                tokio::time::sleep(Duration::from_millis(1)).await;
2352            }
2353        })
2354        .await
2355        .unwrap();
2356        let flush_failed = engine.flush().await.is_err();
2357        // A failed committer closes admission first; its accepted PUT must still
2358        // complete before close returns, even though no manifest may reference it.
2359        let upload_was_owned = upload_release_tx.send(()).is_ok();
2360        tokio::time::timeout(
2361            Duration::from_secs(5),
2362            tokio::task::spawn_blocking(move || drop(engine)),
2363        )
2364        .await
2365        .unwrap()
2366        .unwrap();
2367        assert!(flush_failed);
2368        assert!(upload_was_owned);
2369        assert!(
2370            store
2371                .second_upload_completed
2372                .load(std::sync::atomic::Ordering::SeqCst)
2373        );
2374        assert_eq!(store.list("data/").await.unwrap().len(), 2);
2375        let sequencer = Arc::new(
2376            ManifestSequencer::open(store.clone(), "manifest/")
2377                .await
2378                .unwrap(),
2379        );
2380        let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2381        assert!(
2382            reopened
2383                .fetch(&PartitionKey("p".into()), 0, 100)
2384                .await
2385                .unwrap()
2386                .is_empty()
2387        );
2388        assert_eq!(
2389            reopened
2390                .produce(
2391                    PartitionKey("p".into()),
2392                    Bytes::from_static(b"after-reopen"),
2393                    1,
2394                    (),
2395                    Durability::Sequenced
2396                )
2397                .await
2398                .unwrap()
2399                .base_offset,
2400            Some(0)
2401        );
2402        let rows = reopened
2403            .fetch(&PartitionKey("p".into()), 0, 100)
2404            .await
2405            .unwrap();
2406        assert_eq!(rows.len(), 1);
2407        assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2408        assert_eq!(store.list("data/").await.unwrap().len(), 3);
2409        tokio::task::spawn_blocking(move || drop(reopened))
2410            .await
2411            .unwrap();
2412    }
2413
2414    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2415    async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2416        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2417        let (release_tx, release_rx) = oneshot::channel();
2418        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2419            inner: MemoryBlobStore::new(),
2420            entered: entered_tx,
2421            release: Mutex::new(Some(release_rx)),
2422            fail: false,
2423            panic_on_commit: false,
2424        });
2425        let sequencer = Arc::new(
2426            ManifestSequencer::open(blob.clone(), "manifest/")
2427                .await
2428                .unwrap(),
2429        );
2430        let config = FlushConfig {
2431            max_batches: 1,
2432            max_inflight_flushes: 4,
2433            linger: Duration::ZERO,
2434            budget: BudgetConfig {
2435                enabled: false,
2436                ..BudgetConfig::default()
2437            },
2438            ..FlushConfig::default()
2439        };
2440        let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2441        engine
2442            .produce(
2443                PartitionKey("p".into()),
2444                Bytes::from_static(b"first"),
2445                1,
2446                (),
2447                Durability::Buffered,
2448            )
2449            .await
2450            .unwrap();
2451        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2452        engine
2453            .produce(
2454                PartitionKey("p".into()),
2455                Bytes::from_static(b"second"),
2456                1,
2457                (),
2458                Durability::Buffered,
2459            )
2460            .await
2461            .unwrap();
2462        let shared = engine.shared.clone();
2463        let dropping = tokio::task::spawn_blocking(move || drop(engine));
2464        tokio::time::timeout(Duration::from_secs(5), async {
2465            while !shared.queue.lock().unwrap().shutdown {
2466                tokio::time::sleep(Duration::from_millis(1)).await;
2467            }
2468        })
2469        .await
2470        .unwrap();
2471        let premature = dropping.is_finished();
2472        release_tx.send(()).unwrap();
2473        tokio::time::timeout(Duration::from_secs(5), dropping)
2474            .await
2475            .unwrap()
2476            .unwrap();
2477        assert!(!premature, "shutdown returned before manifest durability");
2478        let reopened = Arc::new(
2479            ManifestSequencer::open(blob.clone(), "manifest/")
2480                .await
2481                .unwrap(),
2482        );
2483        let engine = LogEngine::new(blob, reopened, config, "data/");
2484        let rows = engine
2485            .fetch(&PartitionKey("p".into()), 0, 100)
2486            .await
2487            .unwrap();
2488        assert_eq!(
2489            rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2490            vec![b"first".as_slice(), b"second".as_slice()]
2491        );
2492    }
2493
2494    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2495    async fn panicked_committer_closes_admission_and_fails_flush() {
2496        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2497        let (release_tx, release_rx) = oneshot::channel();
2498        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2499            inner: MemoryBlobStore::new(),
2500            entered: entered_tx,
2501            release: Mutex::new(Some(release_rx)),
2502            fail: false,
2503            panic_on_commit: true,
2504        });
2505        let sequencer = Arc::new(
2506            ManifestSequencer::open(blob.clone(), "manifest/")
2507                .await
2508                .unwrap(),
2509        );
2510        let config = FlushConfig {
2511            max_batches: 1,
2512            max_inflight_flushes: 4,
2513            linger: Duration::ZERO,
2514            budget: BudgetConfig {
2515                enabled: false,
2516                ..BudgetConfig::default()
2517            },
2518            ..FlushConfig::default()
2519        };
2520        let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2521        let first = {
2522            let engine = engine.clone();
2523            tokio::spawn(async move {
2524                engine
2525                    .produce(
2526                        PartitionKey("p".into()),
2527                        Bytes::from_static(b"first"),
2528                        1,
2529                        (),
2530                        Durability::Sequenced,
2531                    )
2532                    .await
2533            })
2534        };
2535        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2536        engine
2537            .produce(
2538                PartitionKey("p".into()),
2539                Bytes::from_static(b"later"),
2540                1,
2541                (),
2542                Durability::Buffered,
2543            )
2544            .await
2545            .unwrap();
2546        let flushing = {
2547            let engine = engine.clone();
2548            tokio::spawn(async move { engine.flush().await })
2549        };
2550        release_tx.send(()).unwrap();
2551        assert!(
2552            tokio::time::timeout(Duration::from_secs(5), first)
2553                .await
2554                .unwrap()
2555                .unwrap()
2556                .is_err()
2557        );
2558        assert!(
2559            tokio::time::timeout(Duration::from_secs(5), flushing)
2560                .await
2561                .unwrap()
2562                .unwrap()
2563                .is_err()
2564        );
2565        tokio::time::timeout(Duration::from_secs(5), async {
2566            while !engine.shared.queue.lock().unwrap().shutdown {
2567                tokio::time::sleep(Duration::from_millis(1)).await;
2568            }
2569        })
2570        .await
2571        .unwrap();
2572        assert!(
2573            engine
2574                .produce(
2575                    PartitionKey("p".into()),
2576                    Bytes::from_static(b"rejected"),
2577                    1,
2578                    (),
2579                    Durability::Buffered
2580                )
2581                .await
2582                .is_err()
2583        );
2584        assert!(engine.flush().await.is_err());
2585    }
2586    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2587    async fn committed_reads_continue_during_new_manifest_publication() {
2588        for fail in [false, true] {
2589            let memory = MemoryBlobStore::new();
2590            let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2591            let seed_sequencer = Arc::new(
2592                ManifestSequencer::open(seed_blob.clone(), "manifest/")
2593                    .await
2594                    .unwrap(),
2595            );
2596            let config = FlushConfig {
2597                max_batches: 1,
2598                max_inflight_flushes: 4,
2599                linger: Duration::ZERO,
2600                budget: BudgetConfig {
2601                    enabled: false,
2602                    ..BudgetConfig::default()
2603                },
2604                ..FlushConfig::default()
2605            };
2606            let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2607            seed.produce(
2608                PartitionKey("p".into()),
2609                Bytes::from_static(b"committed"),
2610                1,
2611                (),
2612                Durability::Sequenced,
2613            )
2614            .await
2615            .unwrap();
2616            drop(seed);
2617            let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2618            let (release_tx, release_rx) = oneshot::channel();
2619            let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2620                inner: memory,
2621                entered: entered_tx,
2622                release: Mutex::new(Some(release_rx)),
2623                fail,
2624                panic_on_commit: false,
2625            });
2626            let sequencer = Arc::new(
2627                ManifestSequencer::open(blob.clone(), "manifest/")
2628                    .await
2629                    .unwrap(),
2630            );
2631            let engine = Arc::new(LogEngine::new(
2632                blob.clone(),
2633                sequencer.clone(),
2634                config,
2635                "data/",
2636            ));
2637            let producing = {
2638                let engine = engine.clone();
2639                tokio::spawn(async move {
2640                    engine
2641                        .produce(
2642                            PartitionKey("p".into()),
2643                            Bytes::from_static(b"new"),
2644                            1,
2645                            (),
2646                            Durability::Sequenced,
2647                        )
2648                        .await
2649                })
2650            };
2651            entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2652            let mut reading = tokio::task::spawn_blocking(move || {
2653                let p = PartitionKey("p".into());
2654                (
2655                    sequencer.high_watermark(&p).unwrap(),
2656                    sequencer.lookup(&p, 0).unwrap(),
2657                    sequencer.snapshot(),
2658                )
2659            });
2660            let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2661            let early_ack = producing.is_finished();
2662            // Release even on regression failure so all worker threads can drain.
2663            release_tx.send(()).unwrap();
2664            let outcome = producing.await.unwrap();
2665            assert_eq!(outcome.is_err(), fail);
2666            let reads_progressed = observed.is_ok();
2667            let (watermark, entries, snapshot) = match observed {
2668                Ok(result) => result.unwrap(),
2669                Err(_) => reading.await.unwrap(),
2670            };
2671            assert!(!early_ack);
2672            assert!(
2673                reads_progressed,
2674                "durable manifest publication blocked committed index reads"
2675            );
2676            assert_eq!(watermark, 1);
2677            assert_eq!(entries.len(), 1);
2678            assert_eq!(snapshot.manifest_count, 1);
2679            let rows = engine
2680                .fetch(&PartitionKey("p".into()), 0, 100)
2681                .await
2682                .unwrap();
2683            assert_eq!(rows.len(), if fail { 1 } else { 2 });
2684            assert_eq!(rows[0].payload.as_ref(), b"committed");
2685            drop(engine);
2686            let reopened = Arc::new(
2687                ManifestSequencer::open(blob.clone(), "manifest/")
2688                    .await
2689                    .unwrap(),
2690            );
2691            let engine = LogEngine::new(blob, reopened, config, "data/");
2692            assert_eq!(
2693                engine
2694                    .fetch(&PartitionKey("p".into()), 0, 100)
2695                    .await
2696                    .unwrap()
2697                    .len(),
2698                if fail { 1 } else { 2 }
2699            );
2700        }
2701    }
2702    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2703    async fn manifest_mutations_remain_serialized_while_readers_progress() {
2704        let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2705        let (release_tx, release_rx) = oneshot::channel();
2706        let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2707            inner: MemoryBlobStore::new(),
2708            entered: entered_tx,
2709            release: Mutex::new(Some(release_rx)),
2710            fail: false,
2711            panic_on_commit: false,
2712        });
2713        let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2714        let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2715            tokio::task::spawn_blocking(move || {
2716                sequencer.commit(&[CommitBatch {
2717                    partition: PartitionKey("p".into()),
2718                    record_count: 1,
2719                    location: BatchLocation {
2720                        object_id: key.into(),
2721                        byte_start: 0,
2722                        byte_len: 1,
2723                    },
2724                    meta: &(),
2725                    epoch: 0,
2726                }])
2727            })
2728        };
2729        let first = commit(sequencer.clone(), "data/first");
2730        entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2731        let second = commit(sequencer.clone(), "data/second");
2732        let truncating = {
2733            let sequencer = sequencer.clone();
2734            tokio::task::spawn_blocking(move || {
2735                sequencer.truncate_before(&PartitionKey("p".into()), 1)
2736            })
2737        };
2738        let reading = {
2739            let sequencer = sequencer.clone();
2740            tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2741        };
2742        let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2743        release_tx.send(()).unwrap();
2744        assert_eq!(
2745            observed.expect("committed reads blocked").unwrap().unwrap(),
2746            0
2747        );
2748        let first = tokio::time::timeout(Duration::from_secs(5), first)
2749            .await
2750            .unwrap()
2751            .unwrap()
2752            .unwrap();
2753        let second = tokio::time::timeout(Duration::from_secs(5), second)
2754            .await
2755            .unwrap()
2756            .unwrap()
2757            .unwrap();
2758        let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2759            .await
2760            .unwrap()
2761            .unwrap()
2762            .unwrap();
2763        assert_eq!(
2764            first,
2765            vec![CommitOutcome::Assigned {
2766                base_offset: 0,
2767                record_count: 1
2768            }]
2769        );
2770        assert_eq!(
2771            second,
2772            vec![CommitOutcome::Assigned {
2773                base_offset: 1,
2774                record_count: 1
2775            }]
2776        );
2777        assert_eq!(dropped, vec!["data/first".to_string()]);
2778        let p = PartitionKey("p".into());
2779        assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2780        assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2781        let entries = sequencer.lookup(&p, 0).unwrap();
2782        assert_eq!(entries.len(), 1);
2783        assert_eq!(entries[0].location.object_id, "data/second");
2784        assert_eq!(sequencer.snapshot().manifest_count, 2);
2785    }
2786}
2787
2788#[cfg(test)]
2789mod idle_batch_wait_tests {
2790    use super::*;
2791    use std::sync::mpsc;
2792
2793    fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2794        Arc::new(Shared {
2795            queue: Mutex::new(Queue {
2796                items: VecDeque::new(),
2797                bytes: 0,
2798                bytes_in_use: 0,
2799                shutdown: false,
2800                next_seq: 1,
2801                completed_through: 0,
2802                first_failed: None,
2803                force_flush: false,
2804                flush_waiters: Vec::new(),
2805                last_enqueue: None,
2806                oldest_enqueue: None,
2807            }),
2808            cv: Condvar::new(),
2809            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2810            budget: Mutex::new(BudgetRuntime::new(config.budget)),
2811            flush_config: config,
2812        })
2813    }
2814
2815    #[test]
2816    fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2817        for linger in [Duration::ZERO, Duration::from_millis(1)] {
2818            for shutdown in [false, true] {
2819                let config = FlushConfig {
2820                    linger,
2821                    ..FlushConfig::default()
2822                };
2823                let shared = empty_shared(config);
2824                let (started_tx, started_rx) = mpsc::channel();
2825                let (result_tx, result_rx) = mpsc::channel();
2826                let worker_shared = shared.clone();
2827                let worker = std::thread::spawn(move || {
2828                    started_tx.send(()).unwrap();
2829                    assert!(
2830                        result_tx
2831                            .send(take_batch(&worker_shared, config, true))
2832                            .is_ok()
2833                    );
2834                });
2835                started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2836                let early = result_rx.recv_timeout(Duration::from_millis(100));
2837                let returned_while_idle = early.is_ok();
2838                {
2839                    let mut q = shared.queue.lock().unwrap();
2840                    if shutdown {
2841                        q.shutdown = true;
2842                    } else {
2843                        let payload = Bytes::from_static(b"first-after-idle");
2844                        q.bytes = payload.len();
2845                        q.bytes_in_use = payload.len();
2846                        q.next_seq = 2;
2847                        q.last_enqueue = Some(Instant::now());
2848                        q.oldest_enqueue = q.last_enqueue;
2849                        q.force_flush = true;
2850                        q.items.push_back(Pending {
2851                            partition: PartitionKey("p".into()),
2852                            record_count: 1,
2853                            payload,
2854                            meta: (),
2855                            epoch: 0,
2856                            durability: Durability::Buffered,
2857                            responder: None,
2858                            seq: 1,
2859                            owner: None,
2860                            expected_head: None,
2861                        });
2862                    }
2863                    shared.cv.notify_all();
2864                }
2865                let result = early
2866                    .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2867                    .unwrap();
2868                worker.join().unwrap();
2869                assert!(
2870                    !returned_while_idle,
2871                    "idle worker polled with linger={linger:?}"
2872                );
2873                match result {
2874                    TakeBatch::Shutdown => assert!(shutdown),
2875                    TakeBatch::Batch(items) => {
2876                        assert!(!shutdown);
2877                        assert_eq!(items.len(), 1);
2878                        assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2879                        assert_eq!(items[0].seq, 1);
2880                    }
2881                    TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2882                }
2883            }
2884        }
2885    }
2886}
2887
2888#[cfg(test)]
2889mod append_owner_tests {
2890    use super::*;
2891    use crate::{InMemorySequencer, MemoryBlobStore};
2892    use async_trait::async_trait;
2893    use std::sync::atomic::{AtomicUsize, Ordering};
2894    struct Lifetime(Arc<AtomicUsize>);
2895    impl Drop for Lifetime {
2896        fn drop(&mut self) {
2897            self.0.fetch_add(1, Ordering::SeqCst);
2898        }
2899    }
2900    struct CopiedPausedPut {
2901        inner: MemoryBlobStore,
2902        entered: std::sync::mpsc::Sender<usize>,
2903        release: tokio::sync::watch::Sender<bool>,
2904    }
2905    #[async_trait]
2906    impl BlobStore for CopiedPausedPut {
2907        async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
2908            self.put_chunks(key, vec![value]).await
2909        }
2910        async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
2911            let copied = Bytes::from(
2912                chunks
2913                    .iter()
2914                    .flat_map(|chunk| chunk.iter().copied())
2915                    .collect::<Vec<_>>(),
2916            );
2917            drop(chunks);
2918            let mut release = self.release.subscribe();
2919            self.entered.send(copied.len()).unwrap();
2920            release.wait_for(|ready| *ready).await.unwrap();
2921            self.inner.put(key, copied).await
2922        }
2923        async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
2924            self.inner.get(key).await
2925        }
2926        async fn get_range(
2927            &self,
2928            key: &str,
2929            range: std::ops::Range<u64>,
2930        ) -> Result<Option<Bytes>, ObjectLogError> {
2931            self.inner.get_range(key, range).await
2932        }
2933        async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
2934            self.inner.list(prefix).await
2935        }
2936        async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
2937            self.inner.delete(key).await
2938        }
2939    }
2940    struct ReentrantLifetime {
2941        shared: std::sync::Weak<Shared<()>>,
2942        unlocked: Arc<AtomicUsize>,
2943    }
2944    impl Drop for ReentrantLifetime {
2945        fn drop(&mut self) {
2946            let shared = self.shared.upgrade().unwrap();
2947            let available = !matches!(
2948                shared.queue.try_lock(),
2949                Err(std::sync::TryLockError::WouldBlock)
2950            );
2951            self.unlocked
2952                .store(usize::from(available), Ordering::SeqCst);
2953        }
2954    }
2955    #[test]
2956    fn rejected_queued_owner_drops_outside_engine_lock() {
2957        let config = FlushConfig::default();
2958        let shared = Arc::new(Shared {
2959            queue: Mutex::new(Queue {
2960                items: VecDeque::new(),
2961                bytes: 0,
2962                bytes_in_use: 0,
2963                shutdown: false,
2964                next_seq: 1,
2965                completed_through: 0,
2966                first_failed: None,
2967                force_flush: false,
2968                flush_waiters: Vec::new(),
2969                last_enqueue: None,
2970                oldest_enqueue: None,
2971            }),
2972            cv: Condvar::new(),
2973            max_buffered_bytes: config.max_buffered_bytes,
2974            budget: Mutex::new(BudgetRuntime::new(config.budget)),
2975            flush_config: config,
2976        });
2977        let unlocked = Arc::new(AtomicUsize::new(0));
2978        let owner = Arc::new(ReentrantLifetime {
2979            shared: Arc::downgrade(&shared),
2980            unlocked: unlocked.clone(),
2981        });
2982        shared.queue.lock().unwrap().items.push_back(Pending {
2983            partition: PartitionKey("p".into()),
2984            record_count: 1,
2985            payload: Bytes::from_static(b"queued"),
2986            meta: (),
2987            epoch: 0,
2988            durability: Durability::Sequenced,
2989            responder: None,
2990            seq: 1,
2991            owner: Some(owner),
2992            expected_head: None,
2993        });
2994        let uploads = fail_engine_and_take_pending_uploads(
2995            &shared,
2996            &mut VecDeque::new(),
2997            1,
2998            ObjectLogError::Sequencer("failure".into()),
2999        );
3000        assert!(uploads.is_empty());
3001        assert_eq!(unlocked.load(Ordering::SeqCst), 1);
3002    }
3003    #[test]
3004    fn failed_engine_keeps_owner_until_copied_pending_upload_finishes() {
3005        let runtime = tokio::runtime::Runtime::new().unwrap();
3006        let (entered, started) = std::sync::mpsc::channel();
3007        let blob = Arc::new(CopiedPausedPut {
3008            inner: MemoryBlobStore::new(),
3009            entered,
3010            release: tokio::sync::watch::channel(false).0,
3011        });
3012        let erased: Arc<dyn BlobStore> = blob.clone();
3013        let engine = LogEngine::new(
3014            erased.clone(),
3015            Arc::new(InMemorySequencer::new()),
3016            FlushConfig::default(),
3017            "idle/",
3018        );
3019        let drops = Arc::new(AtomicUsize::new(0));
3020        let (tx, mut rx) = oneshot::channel();
3021        let batch = vec![
3022            Pending {
3023                partition: PartitionKey("p".into()),
3024                record_count: 1,
3025                payload: Bytes::from_static(b"first"),
3026                meta: (),
3027                epoch: 0,
3028                durability: Durability::Sequenced,
3029                responder: Some(tx),
3030                seq: 1,
3031                owner: Some(Arc::new(Lifetime(drops.clone()))),
3032                expected_head: None,
3033            },
3034            Pending {
3035                partition: PartitionKey("p".into()),
3036                record_count: 1,
3037                payload: Bytes::from_static(b"second"),
3038                meta: (),
3039                epoch: 0,
3040                durability: Durability::Sequenced,
3041                responder: None,
3042                seq: 2,
3043                owner: None,
3044                expected_head: None,
3045            },
3046        ];
3047        let work = start_flush_work(&runtime, &erased, "data/", "", 1, batch, true);
3048        assert_eq!(started.recv_timeout(Duration::from_secs(5)).unwrap(), 11);
3049        let mut pending = VecDeque::from([work]);
3050        let uploads = fail_engine_and_take_pending_uploads(
3051            &engine.shared,
3052            &mut pending,
3053            1,
3054            ObjectLogError::Sequencer("injected committer failure".into()),
3055        );
3056        assert!(rx.try_recv().unwrap().is_err());
3057        assert!(pending.is_empty());
3058        assert_eq!(uploads.len(), 1);
3059        assert_eq!(drops.load(Ordering::SeqCst), 0);
3060        blob.release.send_replace(true);
3061        runtime.block_on(async {
3062            for upload in uploads {
3063                upload.await.unwrap().unwrap();
3064            }
3065        });
3066        assert_eq!(drops.load(Ordering::SeqCst), 1);
3067        assert_eq!(
3068            runtime
3069                .block_on(blob.inner.get("data/00000000000000000001"))
3070                .unwrap()
3071                .unwrap(),
3072            Bytes::from_static(b"firstsecond")
3073        );
3074    }
3075    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3076    async fn cancelled_producer_keeps_owner_until_copied_upload_and_flush_finish() {
3077        let (entered, started) = std::sync::mpsc::channel();
3078        let blob = Arc::new(CopiedPausedPut {
3079            inner: MemoryBlobStore::new(),
3080            entered,
3081            release: tokio::sync::watch::channel(false).0,
3082        });
3083        let engine = Arc::new(LogEngine::new(
3084            blob.clone(),
3085            Arc::new(InMemorySequencer::new()),
3086            FlushConfig {
3087                linger: Duration::ZERO,
3088                ..Default::default()
3089            },
3090            "data/",
3091        ));
3092        let drops = Arc::new(AtomicUsize::new(0));
3093        let owner: AppendOwner = Arc::new(Lifetime(drops.clone()));
3094        let writer = engine.clone();
3095        let caller = tokio::spawn(async move {
3096            writer
3097                .produce_owned_at_epoch(
3098                    PartitionKey("p".into()),
3099                    Bytes::from_static(b"payload"),
3100                    1,
3101                    (),
3102                    Durability::Sequenced,
3103                    0,
3104                    owner,
3105                )
3106                .await
3107        });
3108        assert_eq!(started.recv_timeout(Duration::from_secs(5)).unwrap(), 7);
3109        caller.abort();
3110        let _ = caller.await;
3111        assert_eq!(drops.load(Ordering::SeqCst), 0);
3112        blob.release.send_replace(true);
3113        engine.flush().await.unwrap();
3114        tokio::time::timeout(Duration::from_secs(5), async {
3115            while drops.load(Ordering::SeqCst) == 0 {
3116                tokio::task::yield_now().await;
3117            }
3118        })
3119        .await
3120        .unwrap();
3121        assert_eq!(drops.load(Ordering::SeqCst), 1);
3122    }
3123}