Skip to main content

object_log/
engine.rs

1//! The buffered, multiplexing log engine.
2
3use crate::budget::{
4    BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8    BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21/// The durability point a [`LogEngine::produce`] call resolves at.
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24    /// Resolve as soon as the batch is buffered (fire-and-forget; may be lost on
25    /// crash before the flush). No offset is returned.
26    Buffered,
27    /// Resolve once the containing object is durably PUT (survives crash). No
28    /// offset yet — the commit has not run.
29    Durable,
30    /// Resolve once the batch is durably PUT **and** sequenced (has a stable
31    /// offset). This is the strong, no-loss level.
32    Sequenced,
33}
34
35/// Flush-trigger policy for the engine's group-commit buffer.
36#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38    /// Hard ceiling on one sealed object's payload size (memory / object-store
39    /// physics). **Not** the primary packing control: under normal load packing
40    /// is `ingest_rate × linger`. Keep this high enough that **linger binds**
41    /// before size (default **1 GiB**). Lower only for tight RAM or backend caps.
42    pub max_bytes: usize,
43    /// Flush once this many batches are buffered (secondary ceiling).
44    pub max_batches: usize,
45    /// **Maximum** time a batch may wait before a deadline flush (hard ceiling).
46    /// This is the latency↔throughput control surface: longer wait ⇒ more bytes
47    /// per seal ⇒ fewer durable ops/s. The budget controller may use a shorter
48    /// *effective* linger when media is idle (TD-004). `ZERO` = seal as soon as
49    /// any data is buffered (no co-buffer wait). Default `50ms`.
50    pub linger: Duration,
51    /// Max sealed objects PUT concurrently. Default **1** (single-flight bulk
52    /// path). Raise for parallel S3 PUTs.
53    pub max_inflight_flushes: usize,
54    /// Max bytes in the mutable queue plus in-flight seals. Producers block when
55    /// exceeded. Default **2 GiB**.
56    pub max_buffered_bytes: usize,
57    /// Durable-ops budget controller (default on for Fjord).
58    pub budget: BudgetConfig,
59}
60
61impl Default for FlushConfig {
62    fn default() -> Self {
63        Self {
64            // Physics/safety ceiling so linger defines segment size under load.
65            max_bytes: 1024 * 1024 * 1024,
66            max_batches: 10_000,
67            linger: Duration::from_millis(50),
68            // 1 = single-flight seal (best Local bulk); raise for parallel S3 PUTs.
69            max_inflight_flushes: 1,
70            max_buffered_bytes: 2 * 1024 * 1024 * 1024,
71            budget: BudgetConfig::default(),
72        }
73    }
74}
75
76/// Outcome of a [`LogEngine::produce`] call.
77#[derive(Clone, Debug, PartialEq, Eq)]
78pub struct AppendOutcome {
79    /// First assigned offset, when resolved at [`Durability::Sequenced`].
80    pub base_offset: Option<i64>,
81    /// Last assigned offset, when resolved at [`Durability::Sequenced`].
82    pub last_offset: Option<i64>,
83    /// Whether the batch is durably stored.
84    pub durable: bool,
85    /// Whether the batch has been sequenced (has an offset).
86    pub sequenced: bool,
87}
88
89/// A batch read back by [`LogEngine::fetch`], with its assigned base offset.
90#[derive(Clone, Debug, PartialEq, Eq)]
91pub struct FetchedBatch {
92    /// First offset of the batch.
93    pub base_offset: i64,
94    /// Number of records in the batch.
95    pub record_count: i32,
96    /// The opaque batch payload.
97    pub payload: Bytes,
98}
99
100/// Snapshot of the engine's buffering envelope.
101#[derive(Clone, Debug, PartialEq, Eq)]
102pub struct BufferStats {
103    /// Bytes still waiting in the mutable accumulation queue.
104    pub queued_bytes: usize,
105    /// Bytes owned by queued plus sealed in-flight flush work.
106    pub bytes_in_use: usize,
107    /// Batches still waiting in the mutable accumulation queue.
108    pub queued_batches: usize,
109    /// Configured upper bound for `bytes_in_use`.
110    pub max_buffered_bytes: usize,
111}
112
113type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
114
115fn retryable_storage_error(err: &ObjectLogError) -> bool {
116    matches!(err, ObjectLogError::StorageUnavailable(_))
117}
118
119async fn retry_delay(attempt: usize) {
120    let multiplier = 1u32 << attempt.min(5);
121    tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
122}
123
124async fn put_chunks_with_retries(
125    blob: &Arc<dyn BlobStore>,
126    key: &str,
127    chunks: Vec<Bytes>,
128) -> Result<(), ObjectLogError> {
129    // `Bytes::clone` is refcount-only; Local put_chunks streams without re-merge.
130    let mut attempt = 0usize;
131    loop {
132        match blob.put_chunks(key, chunks.clone()).await {
133            Ok(()) => return Ok(()),
134            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
135                retry_delay(attempt).await;
136                attempt += 1;
137            }
138            Err(err) => return Err(err),
139        }
140    }
141}
142
143async fn get_range_with_retries(
144    blob: &Arc<dyn BlobStore>,
145    key: &str,
146    range: std::ops::Range<u64>,
147) -> Result<Option<Bytes>, ObjectLogError> {
148    let mut attempt = 0usize;
149    loop {
150        match blob.get_range(key, range.clone()).await {
151            Ok(bytes) => return Ok(bytes),
152            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
153                retry_delay(attempt).await;
154                attempt += 1;
155            }
156            Err(err) => return Err(err),
157        }
158    }
159}
160
161struct Pending<M> {
162    partition: PartitionKey,
163    record_count: i32,
164    payload: Bytes,
165    meta: M,
166    durability: Durability,
167    responder: Option<Responder>,
168    /// Monotonic enqueue id for [`LogEngine::flush`] barriers.
169    seq: u64,
170}
171
172struct FlushWork<M> {
173    batch: Vec<Pending<M>>,
174    locations: Vec<BatchLocation>,
175    responders: Vec<(Durability, Option<Responder>)>,
176    bytes: usize,
177    /// Highest enqueue seq included in this flush object.
178    max_seq: u64,
179    put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
180    put_started: Instant,
181    put_result: Option<Result<Duration, ObjectLogError>>,
182}
183
184enum TakeBatch<M> {
185    Batch(Vec<Pending<M>>),
186    Empty,
187    Shutdown,
188}
189
190type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
191
192struct Queue<M> {
193    items: VecDeque<Pending<M>>,
194    bytes: usize,
195    bytes_in_use: usize,
196    shutdown: bool,
197    /// Next seq to assign on enqueue.
198    next_seq: u64,
199    /// Highest seq fully sealed (put + sequencer commit finished).
200    completed_through: u64,
201    /// When true, take_batch flushes even if under size/linger.
202    force_flush: bool,
203    /// Waiters: (barrier_seq inclusive, responder).
204    flush_waiters: Vec<FlushWaiter>,
205    /// Last successful produce enqueue (for idle early-flush).
206    last_enqueue: Option<Instant>,
207    /// Enqueue time of the front item (linger deadline = oldest + linger).
208    oldest_enqueue: Option<Instant>,
209}
210
211struct Shared<M> {
212    queue: Mutex<Queue<M>>,
213    cv: Condvar,
214    max_buffered_bytes: usize,
215    /// Budget controller + inspectable counters (TD-004).
216    budget: Mutex<BudgetRuntime>,
217    /// Max linger (operator ceiling) and budget config copy for the flush loop.
218    flush_config: FlushConfig,
219}
220
221/// A buffered, multiplexing append-log engine over a [`BlobStore`], with
222/// sequencing delegated to a [`Sequencer`].
223///
224/// `produce` group-commits: many batches across many partitions are multiplexed
225/// into one object, PUT durably, then handed to the sequencer in a single call —
226/// so PUT count is decoupled from produce count. A single flush worker preserves
227/// per-[`PartitionKey`] arrival order and never splits a partition across
228/// concurrent commits.
229pub struct LogEngine<S: Sequencer> {
230    shared: Arc<Shared<S::Meta>>,
231    blob: Arc<dyn BlobStore>,
232    sequencer: Arc<S>,
233    /// Data-object key prefix (`<prefix><counter:020>`). Used by orphan reaping.
234    key_prefix: String,
235    flush_thread: Option<JoinHandle<()>>,
236}
237
238impl<S> LogEngine<S>
239where
240    S: Sequencer + 'static,
241    S::Meta: Send + 'static,
242{
243    /// Create an engine over `blob` and `sequencer` with the given flush policy.
244    /// Objects are keyed `<key_prefix><counter>`; pick a prefix unique to this
245    /// engine instance if several share a store.
246    pub fn new(
247        blob: Arc<dyn BlobStore>,
248        sequencer: Arc<S>,
249        config: FlushConfig,
250        key_prefix: impl Into<String>,
251    ) -> Self {
252        if let Err(msg) = config.budget.validate() {
253            panic!("invalid FlushConfig.budget: {msg}");
254        }
255        if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
256            eprintln!("object-log flush config: {config:?}");
257        }
258        let budget_rt = BudgetRuntime::new(config.budget);
259        let shared = Arc::new(Shared {
260            queue: Mutex::new(Queue {
261                items: VecDeque::new(),
262                bytes: 0,
263                bytes_in_use: 0,
264                shutdown: false,
265                next_seq: 1, // first enqueued batch gets seq 1
266                completed_through: 0,
267                force_flush: false,
268                flush_waiters: Vec::new(),
269                last_enqueue: None,
270                oldest_enqueue: None,
271            }),
272            cv: Condvar::new(),
273            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
274            budget: Mutex::new(budget_rt),
275            flush_config: config,
276        });
277        let key_prefix = key_prefix.into();
278        let flush_thread = {
279            let shared = Arc::clone(&shared);
280            let blob = Arc::clone(&blob);
281            let sequencer = Arc::clone(&sequencer);
282            let prefix = key_prefix.clone();
283            std::thread::Builder::new()
284                .name("object-log-flush".into())
285                .spawn(move || flush_loop(shared, blob, sequencer, config, prefix))
286                .expect("spawn flush thread")
287        };
288        Self {
289            shared,
290            blob,
291            sequencer,
292            key_prefix,
293            flush_thread: Some(flush_thread),
294        }
295    }
296
297    /// Prefix used for data objects written by this engine.
298    pub fn data_prefix(&self) -> &str {
299        &self.key_prefix
300    }
301
302    /// Borrow the engine's blob store (e.g. for orphan reaping or inspection).
303    pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
304        &self.blob
305    }
306
307    /// Borrow the engine's sequencer.
308    pub fn sequencer(&self) -> &Arc<S> {
309        &self.sequencer
310    }
311
312    /// Buffer a batch and resolve at the requested [`Durability`].
313    pub async fn produce(
314        &self,
315        partition: PartitionKey,
316        payload: Bytes,
317        record_count: i32,
318        meta: S::Meta,
319        durability: Durability,
320    ) -> Result<AppendOutcome, ObjectLogError> {
321        if record_count <= 0 {
322            return Err(ObjectLogError::InvalidBatch(
323                "record_count must be > 0".into(),
324            ));
325        }
326        if matches!(durability, Durability::Buffered) {
327            self.enqueue(Pending {
328                partition,
329                record_count,
330                payload,
331                meta,
332                durability,
333                responder: None,
334                seq: 0, // filled in enqueue
335            })?;
336            return Ok(AppendOutcome {
337                base_offset: None,
338                last_offset: None,
339                durable: false,
340                sequenced: false,
341            });
342        }
343        // fail_closed: reserve predicted media ops before waiting on flush.
344        {
345            let mut budget = self.shared.budget.lock().expect("poisoned");
346            if budget.config.enabled
347                && budget.config.mode == BudgetMode::FailClosed
348                && !budget.reserve_for_fail_closed(Instant::now())
349            {
350                return Err(ObjectLogError::BudgetExceeded(
351                    "insufficient durable-ops budget to admit produce".into(),
352                ));
353            }
354        }
355        // budget_priority: wait briefly for tokens when empty.
356        if self.shared.flush_config.budget.enabled
357            && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
358        {
359            let timeout = self.shared.flush_config.budget.admission_timeout;
360            let deadline = Instant::now() + timeout;
361            loop {
362                let mut budget = self.shared.budget.lock().expect("poisoned");
363                if budget.can_admit_now(Instant::now()) {
364                    break;
365                }
366                drop(budget);
367                if Instant::now() >= deadline {
368                    return Err(ObjectLogError::BudgetExceeded(
369                        "timed out waiting for durable-ops budget".into(),
370                    ));
371                }
372                std::thread::sleep(Duration::from_millis(1));
373            }
374        }
375        let (tx, rx) = oneshot::channel();
376        self.enqueue(Pending {
377            partition,
378            record_count,
379            payload,
380            meta,
381            durability,
382            responder: Some(tx),
383            seq: 0,
384        })?;
385        rx.await
386            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
387    }
388
389    /// Seal every batch enqueued **at or before** this call (barrier).
390    ///
391    /// Use after [`Durability::Buffered`] produces to wait for durable PUT (and
392    /// sequencing) of that prior work. Concurrent produces enqueued after this
393    /// call are not required to finish. Empty buffer returns immediately.
394    pub async fn flush(&self) -> Result<(), ObjectLogError> {
395        let rx = {
396            let mut q = self.shared.queue.lock().expect("poisoned");
397            if q.shutdown {
398                return Err(ObjectLogError::Sequencer("engine shutting down".into()));
399            }
400            if q.next_seq == 1 {
401                // Nothing has ever been enqueued.
402                return Ok(());
403            }
404            let barrier = q.next_seq - 1;
405            if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
406                return Ok(());
407            }
408            let (tx, rx) = oneshot::channel();
409            q.force_flush = true;
410            q.flush_waiters.push((barrier, tx));
411            self.shared.cv.notify_all();
412            rx
413        };
414        rx.await
415            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
416    }
417
418    fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
419        let item_len = item.payload.len();
420        let max_buffered_bytes = self.shared.max_buffered_bytes;
421        let shared = Arc::clone(&self.shared);
422        let mut q = shared.queue.lock().expect("poisoned");
423        while !q.shutdown
424            && q.bytes_in_use > 0
425            && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
426        {
427            q = shared.cv.wait(q).expect("poisoned");
428        }
429        if q.shutdown {
430            return Err(ObjectLogError::Sequencer("engine shutting down".into()));
431        }
432        let now = Instant::now();
433        item.seq = q.next_seq;
434        q.next_seq = q.next_seq.saturating_add(1);
435        q.bytes += item_len;
436        q.bytes_in_use += item_len;
437        q.last_enqueue = Some(now);
438        if q.oldest_enqueue.is_none() {
439            q.oldest_enqueue = Some(now);
440        }
441        q.items.push_back(item);
442        shared.cv.notify_all();
443        Ok(())
444    }
445
446    /// Read batches covering offsets at/after `offset`, up to ~`max_bytes`.
447    pub async fn fetch(
448        &self,
449        partition: &PartitionKey,
450        offset: i64,
451        max_bytes: usize,
452    ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
453        let entries = self.sequencer.lookup(partition, offset)?;
454        let mut out = Vec::new();
455        let mut total = 0usize;
456        for e in entries {
457            if total >= max_bytes && !out.is_empty() {
458                break;
459            }
460            let batch = self.load_entry(&e).await?;
461            total += batch.payload.len();
462            out.push(batch);
463        }
464        Ok(out)
465    }
466
467    /// Stream batches from `offset` onward without materializing a full `Vec`.
468    ///
469    /// Calls `visit` once per index entry in order. Suitable for wide offset
470    /// windows (bounded-RAM replay). Stops and returns the error if `visit`
471    /// fails. Unlike [`fetch`], there is no byte budget — visit every remaining
472    /// batch (or stop yourself inside `visit`).
473    pub async fn fetch_stream<F>(
474        &self,
475        partition: &PartitionKey,
476        offset: i64,
477        mut visit: F,
478    ) -> Result<(), ObjectLogError>
479    where
480        F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
481    {
482        let entries = self.sequencer.lookup(partition, offset)?;
483        for e in entries {
484            visit(self.load_entry(&e).await?)?;
485        }
486        Ok(())
487    }
488
489    async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
490        let start = e.location.byte_start as u64;
491        let end = start + e.location.byte_len as u64;
492        let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
493            .await?
494            .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
495        Ok(FetchedBatch {
496            base_offset: e.base_offset,
497            record_count: e.record_count,
498            payload: bytes,
499        })
500    }
501
502    /// Drop the partition's log below `offset` and delete any object that thereby
503    /// becomes fully unreferenced.
504    pub async fn truncate_before(
505        &self,
506        partition: &PartitionKey,
507        offset: i64,
508    ) -> Result<(), ObjectLogError> {
509        let dead = self.sequencer.truncate_before(partition, offset)?;
510        for object_id in dead {
511            self.blob.delete(&object_id).await?;
512        }
513        Ok(())
514    }
515
516    /// Delete data-prefix objects that are not in `live`.
517    ///
518    /// **Safety:** call only when this engine (and any other writer using the
519    /// same prefix) is quiescent — otherwise an in-flight put that has not yet
520    /// been committed may be deleted as an "orphan". Prefer running offline or
521    /// after drop. Does not touch keys outside [`data_prefix`](Self::data_prefix)
522    /// (e.g. a separate manifest prefix is safe).
523    ///
524    /// Returns the deleted object keys.
525    pub async fn reap_orphans(
526        &self,
527        live: &HashSet<String>,
528    ) -> Result<Vec<String>, ObjectLogError> {
529        reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
530    }
531
532    /// Return a point-in-time snapshot of queued and in-flight payload bytes.
533    pub fn buffer_stats(&self) -> BufferStats {
534        let q = self.shared.queue.lock().expect("poisoned");
535        BufferStats {
536            queued_bytes: q.bytes,
537            bytes_in_use: q.bytes_in_use,
538            queued_batches: q.items.len(),
539            max_buffered_bytes: self.shared.max_buffered_bytes,
540        }
541    }
542
543    /// Inspect budget controller layers and counters (TD-004).
544    pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
545        let now = Instant::now();
546        let mut budget = self.shared.budget.lock().expect("poisoned");
547        budget.refill(now);
548        let effective = budget.effective_budget_per_sec;
549        let reason = if !budget.config.enabled {
550            EffectiveReason::Disabled
551        } else if budget.config.budget_per_sec_cap.is_some()
552            && budget
553                .config
554                .budget_per_sec_cap
555                .is_some_and(|c| (effective - c).abs() < 1e-9)
556        {
557            EffectiveReason::ConfigCap
558        } else if budget.ongoing_capacity.is_some() {
559            EffectiveReason::Ongoing
560        } else if budget.startup_capacity.is_some() {
561            EffectiveReason::StartupProbe
562        } else {
563            EffectiveReason::DefaultCapacity
564        };
565        let (queued, last_enq) = {
566            let q = self.shared.queue.lock().expect("poisoned");
567            (q.bytes, q.last_enqueue)
568        };
569        let early = budget.allow_early_flush(now, queued, last_enq);
570        let max_linger = self.shared.flush_config.linger;
571        let eff_linger = if !budget.config.enabled {
572            max_linger
573        } else if early || max_linger.is_zero() {
574            Duration::ZERO
575        } else {
576            max_linger
577        };
578        PipelineSnapshot {
579            budget_per_sec: EffectiveKnob {
580                configured: budget.config.budget_per_sec_cap,
581                startup_measured: budget.startup_capacity,
582                ongoing_measured: budget.ongoing_capacity,
583                effective,
584                reason,
585            },
586            effective_linger_ms: EffectiveKnob {
587                configured: Some(max_linger.as_millis() as u64),
588                startup_measured: None,
589                ongoing_measured: None,
590                effective: eff_linger.as_millis() as u64,
591                reason: if early {
592                    EffectiveReason::Ongoing
593                } else {
594                    EffectiveReason::Configured
595                },
596            },
597            max_linger_ms: max_linger.as_millis() as u64,
598            token_fill_ratio: budget.fill_ratio(),
599            media_ops_total: budget.media_ops_total,
600            overdraft_total: budget.overdraft_total,
601            flushes_total: budget.flushes_total,
602            undersized_deadline_flushes: budget.undersized_deadline_flushes,
603            predicted_media_ops: budget.predicted_media_ops,
604            budget_enabled: budget.config.enabled,
605            budget_mode: budget.config.mode,
606        }
607    }
608}
609
610/// Delete objects under `data_prefix` whose keys are absent from `live`.
611///
612/// See [`LogEngine::reap_orphans`] for safety notes. `live` is typically
613/// [`InMemorySequencer::live_object_ids`](crate::InMemorySequencer::live_object_ids)
614/// or [`ManifestSequencer::live_object_ids`](crate::ManifestSequencer::live_object_ids).
615pub async fn reap_orphans(
616    blob: &dyn BlobStore,
617    data_prefix: &str,
618    live: &HashSet<String>,
619) -> Result<Vec<String>, ObjectLogError> {
620    let keys = blob.list(data_prefix).await?;
621    let mut deleted = Vec::new();
622    for key in keys {
623        if live.contains(&key) {
624            continue;
625        }
626        blob.delete(&key).await?;
627        deleted.push(key);
628    }
629    Ok(deleted)
630}
631
632impl<S: Sequencer> Drop for LogEngine<S> {
633    fn drop(&mut self) {
634        {
635            let mut q = self.shared.queue.lock().expect("poisoned");
636            q.shutdown = true;
637        }
638        self.shared.cv.notify_all();
639        if let Some(t) = self.flush_thread.take() {
640            let _ = t.join();
641        }
642    }
643}
644
645fn flush_loop<S>(
646    shared: Arc<Shared<S::Meta>>,
647    blob: Arc<dyn BlobStore>,
648    sequencer: Arc<S>,
649    config: FlushConfig,
650    prefix: String,
651) where
652    S: Sequencer + 'static,
653    S::Meta: Send + 'static,
654{
655    let max_inflight = config.max_inflight_flushes.max(1);
656    let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
657        .ok()
658        .and_then(|v| v.parse::<usize>().ok())
659        .filter(|v| *v > 0)
660        .unwrap_or_else(|| max_inflight.min(8));
661    let rt = tokio::runtime::Builder::new_multi_thread()
662        .worker_threads(worker_threads)
663        .enable_all()
664        .build()
665        .expect("flush runtime");
666    let mut counter: u64 = 0;
667    let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
668    let mut active_puts = 0usize;
669    let mut shutdown = false;
670
671    loop {
672        while !shutdown && active_puts < max_inflight {
673            let wait_for_more = pending.is_empty();
674            match take_batch(&shared, config, wait_for_more) {
675                TakeBatch::Batch(batch) => {
676                    counter += 1;
677                    let concurrent = max_inflight > 1;
678                    let work = start_flush_work(&rt, &blob, &prefix, counter, batch, concurrent);
679                    if concurrent {
680                        active_puts += 1;
681                    }
682                    pending.push_back(work);
683                }
684                TakeBatch::Empty => break,
685                TakeBatch::Shutdown => shutdown = true,
686            }
687        }
688
689        let mut made_progress = false;
690        for work in pending.iter_mut() {
691            let Some(put) = work.put.as_ref() else {
692                continue;
693            };
694            if !put.is_finished() {
695                continue;
696            }
697            let put = work.put.take().expect("put handle exists");
698            let elapsed = work.put_started.elapsed();
699            work.put_result = Some(match rt.block_on(put) {
700                Ok(Ok(())) => Ok(elapsed),
701                Ok(Err(e)) => Err(e),
702                Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
703                    "flush task failed: {e}"
704                ))),
705            });
706            active_puts = active_puts.saturating_sub(1);
707            made_progress = true;
708        }
709
710        while pending
711            .front()
712            .is_some_and(|work| work.put_result.is_some())
713        {
714            let work = pending.pop_front().expect("front exists");
715            let released = finish_flush_work(&shared, &blob, &sequencer, work);
716            let mut q = shared.queue.lock().expect("poisoned");
717            q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
718            shared.cv.notify_all();
719            made_progress = true;
720        }
721
722        if pending.is_empty() {
723            {
724                let mut q = shared.queue.lock().expect("poisoned");
725                if q.items.is_empty() && q.bytes_in_use == 0 {
726                    // Nothing in flight: any remaining waiters with barrier already
727                    // covered (or no work after last seal) can complete.
728                    notify_flush_waiters(&mut q);
729                }
730            }
731            shared.cv.notify_all();
732            if shutdown {
733                // Fail any remaining flush waiters.
734                let mut q = shared.queue.lock().expect("poisoned");
735                for (_, tx) in q.flush_waiters.drain(..) {
736                    let _ = tx.send(Err(ObjectLogError::Sequencer(
737                        "engine shutting down".into(),
738                    )));
739                }
740                return;
741            }
742            continue;
743        }
744
745        if !made_progress {
746            std::thread::sleep(Duration::from_millis(1));
747        }
748    }
749}
750
751/// How long to sleep before re-evaluating a seal decision.
752/// - `ZERO` means seal now (early-flush or linger deadline hit).
753/// - Otherwise wait at most this long (interrupted by new produces).
754fn effective_linger<M>(
755    shared: &Shared<M>,
756    config: FlushConfig,
757    queued_bytes: usize,
758    last_enqueue: Option<Instant>,
759    oldest_enqueue: Option<Instant>,
760) -> Duration {
761    if config.linger.is_zero() {
762        return Duration::ZERO;
763    }
764    let now = Instant::now();
765    if config.budget.enabled {
766        let mut budget = shared.budget.lock().expect("poisoned");
767        budget.refill(now);
768        if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
769            budget.note_early_flush(now);
770            return Duration::ZERO;
771        }
772    }
773    // True linger: seal when the oldest buffered item reaches max wait.
774    if let Some(oldest) = oldest_enqueue {
775        let deadline = oldest + config.linger;
776        if now >= deadline {
777            return Duration::ZERO;
778        }
779        let until_deadline = deadline.saturating_duration_since(now);
780        // Also wake when idle gate could open, so sparse produces don't wait full linger.
781        if config.budget.enabled
782            && !config.budget.early_flush_idle.is_zero()
783            && let Some(last) = last_enqueue
784        {
785            let idle_at = last + config.budget.early_flush_idle;
786            if idle_at > now && idle_at < deadline {
787                return idle_at.saturating_duration_since(now);
788            }
789        }
790        return until_deadline;
791    }
792    config.linger
793}
794
795fn take_batch<M>(
796    shared: &Arc<Shared<M>>,
797    config: FlushConfig,
798    wait_when_empty: bool,
799) -> TakeBatch<M> {
800    let mut q = shared.queue.lock().expect("poisoned");
801    loop {
802        if q.items.is_empty() {
803            if q.shutdown {
804                return TakeBatch::Shutdown;
805            }
806            if !wait_when_empty {
807                return TakeBatch::Empty;
808            }
809            let linger = {
810                let queued = q.bytes;
811                let last_enq = q.last_enqueue;
812                let oldest = q.oldest_enqueue;
813                drop(q);
814                let l = effective_linger(shared, config, queued, last_enq, oldest);
815                q = shared.queue.lock().expect("poisoned");
816                l
817            };
818            if linger.is_zero() {
819                q = shared.cv.wait(q).expect("poisoned");
820            } else {
821                let (guard, timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
822                q = guard;
823                if timeout.timed_out() && q.items.is_empty() {
824                    return TakeBatch::Empty;
825                }
826            }
827            continue;
828        }
829
830        let linger = {
831            let queued = q.bytes;
832            let last_enq = q.last_enqueue;
833            let oldest = q.oldest_enqueue;
834            drop(q);
835            let l = effective_linger(shared, config, queued, last_enq, oldest);
836            q = shared.queue.lock().expect("poisoned");
837            l
838        };
839
840        let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
841        let force = q.force_flush;
842        let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
843        if triggered {
844            if !size_trigger && !q.shutdown && !q.items.is_empty() {
845                let mut budget = shared.budget.lock().expect("poisoned");
846                if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
847                    budget.undersized_deadline_flushes += 1;
848                }
849            }
850            break;
851        }
852
853        // Short wait (until idle gate or linger deadline); re-evaluate, do not
854        // seal solely because a probe sleep timed out.
855        let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
856        q = guard;
857    }
858
859    let force_drain = q.force_flush;
860    let mut items = Vec::new();
861    let mut bytes = 0usize;
862    // On force_flush, still respect max_bytes so one object stays bounded, but
863    // ignore max_batches so a barrier can drain large queues across objects.
864    while let Some(item) = q.items.pop_front() {
865        q.bytes = q.bytes.saturating_sub(item.payload.len());
866        bytes += item.payload.len();
867        items.push(item);
868        if bytes >= config.max_bytes {
869            break;
870        }
871        if !force_drain && items.len() >= config.max_batches {
872            break;
873        }
874    }
875    q.oldest_enqueue = if q.items.is_empty() {
876        None
877    } else {
878        // Approximate: next seal window starts now for remaining items.
879        Some(Instant::now())
880    };
881    // Keep forcing while a barrier still has queued work.
882    if force_drain {
883        q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
884    }
885    if items.is_empty() {
886        TakeBatch::Empty
887    } else {
888        TakeBatch::Batch(items)
889    }
890}
891
892fn notify_flush_waiters<M>(q: &mut Queue<M>) {
893    let done = q.completed_through;
894    let mut i = 0;
895    while i < q.flush_waiters.len() {
896        if q.flush_waiters[i].0 <= done {
897            let (_, tx) = q.flush_waiters.swap_remove(i);
898            let _ = tx.send(Ok(()));
899        } else {
900            i += 1;
901        }
902    }
903    if q.flush_waiters.is_empty() {
904        q.force_flush = false;
905    } else if q.items.is_empty() && q.bytes_in_use == 0 {
906        // Barriers still waiting but no work — should not happen; clear force.
907        q.force_flush = true;
908    } else {
909        q.force_flush = true;
910    }
911}
912
913fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
914    for (_, tx) in responders.iter_mut() {
915        if let Some(tx) = tx.take() {
916            let _ = tx.send(Err(err.clone()));
917        }
918    }
919}
920
921fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
922    for (durability, tx) in responders.iter_mut() {
923        if *durability == Durability::Durable
924            && let Some(tx) = tx.take()
925        {
926            let _ = tx.send(Ok(AppendOutcome {
927                base_offset: None,
928                last_offset: None,
929                durable: true,
930                sequenced: false,
931            }));
932        }
933    }
934}
935
936fn prepare_flush_work<M>(
937    prefix: &str,
938    counter: u64,
939    mut batch: Vec<Pending<M>>,
940) -> (FlushWork<M>, String, Vec<Bytes>) {
941    let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
942    let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
943    let key = format!("{prefix}{counter:020}");
944    let mut offset = 0usize;
945    for p in &batch {
946        let start = offset as u32;
947        offset += p.payload.len();
948        chunks.push(p.payload.clone());
949        locations.push(BatchLocation {
950            object_id: key.clone(),
951            byte_start: start,
952            byte_len: p.payload.len() as u32,
953        });
954    }
955    if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
956        eprintln!(
957            "object-log seal: key={key} batches={} bytes={offset}",
958            batch.len()
959        );
960    }
961    let responders: Vec<(Durability, Option<Responder>)> = batch
962        .iter_mut()
963        .map(|p| (p.durability, p.responder.take()))
964        .collect();
965    let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
966    let work = FlushWork {
967        batch,
968        locations,
969        responders,
970        bytes: offset,
971        max_seq,
972        put: None,
973        put_started: Instant::now(),
974        put_result: None,
975    };
976    (work, key, chunks)
977}
978
979/// Start a durable put for a sealed batch.
980///
981/// - `concurrent == false` (default `max_inflight_flushes == 1`): `block_on` the
982///   put on the flush thread; Local uses `block_in_place` (no spawn_blocking queue).
983/// - `concurrent == true`: spawn put tasks for parallel S3-style throughput.
984fn start_flush_work<M>(
985    rt: &tokio::runtime::Runtime,
986    blob: &Arc<dyn BlobStore>,
987    prefix: &str,
988    counter: u64,
989    batch: Vec<Pending<M>>,
990    concurrent: bool,
991) -> FlushWork<M> {
992    let (mut work, key, chunks) = prepare_flush_work(prefix, counter, batch);
993    let blob = Arc::clone(blob);
994    let _ = blob.take_media_op_stats();
995    work.put_started = Instant::now();
996    if concurrent {
997        work.put =
998            Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
999    } else {
1000        let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1001        let elapsed = work.put_started.elapsed();
1002        work.put_result = Some(match put_result {
1003            Ok(()) => Ok(elapsed),
1004            Err(e) => Err(e),
1005        });
1006    }
1007    work
1008}
1009
1010fn finish_flush_work<S, M>(
1011    shared: &Arc<Shared<M>>,
1012    blob: &Arc<dyn BlobStore>,
1013    sequencer: &Arc<S>,
1014    mut work: FlushWork<S::Meta>,
1015) -> usize
1016where
1017    S: Sequencer + 'static,
1018    S::Meta: Send + 'static,
1019{
1020    let release_bytes = work.bytes;
1021    // Durable-then-sequence: the object PUT may have overlapped later PUTs, but
1022    // sequencer commits are still completed in object creation order.
1023    let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1024    let put_elapsed = match work.put_result.take().expect("put result is ready") {
1025        Ok(elapsed) => elapsed,
1026        Err(e) => {
1027            send_storage_error(&mut work.responders, e.clone());
1028            let mut q = shared.queue.lock().expect("poisoned");
1029            // Unblock flush waiters covering this seq with the storage error.
1030            let done = work.max_seq;
1031            if done >= q.completed_through {
1032                q.completed_through = done;
1033            }
1034            let mut i = 0;
1035            while i < q.flush_waiters.len() {
1036                if q.flush_waiters[i].0 <= done {
1037                    let (_, tx) = q.flush_waiters.swap_remove(i);
1038                    let _ = tx.send(Err(e.clone()));
1039                } else {
1040                    i += 1;
1041                }
1042            }
1043            shared.cv.notify_all();
1044            return release_bytes;
1045        }
1046    };
1047
1048    // Media ops for the data object put.
1049    let mut media_ops = blob.take_media_op_stats().map(|s| s.media_ops).unwrap_or(1); // fallback: 1 per successful put
1050
1051    // Signal Durable-level waiters now (after PUT, before commit).
1052    send_durable_acks(&mut work.responders);
1053
1054    // Sequence the whole object atomically.
1055    let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1056        .batch
1057        .iter()
1058        .zip(work.locations.iter())
1059        .map(|(p, loc)| CommitBatch {
1060            partition: p.partition.clone(),
1061            record_count: p.record_count,
1062            location: loc.clone(),
1063            meta: &p.meta,
1064        })
1065        .collect();
1066
1067    let commit_started = timing.then(Instant::now);
1068    // Clear stats again so sequencer durable work (e.g. ManifestSequencer put)
1069    // on the shared store is counted into the same budget.
1070    let _ = blob.take_media_op_stats();
1071    match sequencer.commit(&commit_batches) {
1072        Ok(outcomes) => {
1073            if let Some(commit_started) = commit_started {
1074                eprintln!(
1075                    "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={}",
1076                    work.bytes,
1077                    commit_batches.len(),
1078                    put_elapsed.as_millis(),
1079                    commit_started.elapsed().as_millis()
1080                );
1081            }
1082            if let Some(s) = blob.take_media_op_stats() {
1083                media_ops = media_ops.saturating_add(s.media_ops);
1084            }
1085            for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1086                if let Some(tx) = tx.take() {
1087                    let (base, last) = match outcome {
1088                        CommitOutcome::Assigned {
1089                            base_offset,
1090                            record_count,
1091                        } => (
1092                            Some(base_offset),
1093                            Some(base_offset + record_count as i64 - 1),
1094                        ),
1095                        CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1096                    };
1097                    let _ = tx.send(Ok(AppendOutcome {
1098                        base_offset: base,
1099                        last_offset: last,
1100                        durable: true,
1101                        sequenced: true,
1102                    }));
1103                }
1104            }
1105        }
1106        Err(e) => {
1107            if let Some(s) = blob.take_media_op_stats() {
1108                media_ops = media_ops.saturating_add(s.media_ops);
1109            }
1110            send_storage_error(&mut work.responders, e);
1111        }
1112    }
1113
1114    {
1115        let mut budget = shared.budget.lock().expect("poisoned");
1116        budget.consume_after_flush(media_ops, Instant::now());
1117    }
1118    {
1119        let mut q = shared.queue.lock().expect("poisoned");
1120        if work.max_seq >= q.completed_through {
1121            q.completed_through = work.max_seq;
1122        }
1123        notify_flush_waiters(&mut q);
1124    }
1125    shared.cv.notify_all();
1126
1127    release_bytes
1128}