Skip to main content

object_log/
engine.rs

1//! The buffered, multiplexing log engine.
2
3use crate::budget::{
4    BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{BlobStore, CommitBatch, CommitOutcome, ObjectLogError, PartitionKey, Sequencer};
8use bytes::Bytes;
9use std::collections::VecDeque;
10use std::sync::{Arc, Condvar, Mutex};
11use std::thread::JoinHandle;
12use std::time::{Duration, Instant};
13use tokio::sync::oneshot;
14use tokio::task::JoinHandle as TokioJoinHandle;
15
16const STORAGE_RETRY_ATTEMPTS: usize = 5;
17const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
18
19/// The durability point a [`LogEngine::produce`] call resolves at.
20#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub enum Durability {
22    /// Resolve as soon as the batch is buffered (fire-and-forget; may be lost on
23    /// crash before the flush). No offset is returned.
24    Buffered,
25    /// Resolve once the containing object is durably PUT (survives crash). No
26    /// offset yet — the commit has not run.
27    Durable,
28    /// Resolve once the batch is durably PUT **and** sequenced (has a stable
29    /// offset). This is the strong, no-loss level.
30    Sequenced,
31}
32
33/// Flush-trigger policy for the engine's group-commit buffer.
34#[derive(Clone, Copy, Debug)]
35pub struct FlushConfig {
36    /// Hard ceiling on one sealed object's payload size (memory / object-store
37    /// physics). **Not** the primary packing control: under normal load packing
38    /// is `ingest_rate × linger`. Keep this high enough that **linger binds**
39    /// before size (default **1 GiB**). Lower only for tight RAM or backend caps.
40    pub max_bytes: usize,
41    /// Flush once this many batches are buffered (secondary ceiling).
42    pub max_batches: usize,
43    /// **Maximum** time a batch may wait before a deadline flush (hard ceiling).
44    /// This is the latency↔throughput control surface: longer wait ⇒ more bytes
45    /// per seal ⇒ fewer durable ops/s. The budget controller may use a shorter
46    /// *effective* linger when media is idle (TD-004). `ZERO` = seal as soon as
47    /// any data is buffered (no co-buffer wait). Default `50ms`.
48    pub linger: Duration,
49    /// Max sealed objects PUT concurrently. Default **1** (single-flight bulk
50    /// path). Raise for parallel S3 PUTs.
51    pub max_inflight_flushes: usize,
52    /// Max bytes in the mutable queue plus in-flight seals. Producers block when
53    /// exceeded. Default **2 GiB**.
54    pub max_buffered_bytes: usize,
55    /// Durable-ops budget controller (default on for Fjord).
56    pub budget: BudgetConfig,
57}
58
59impl Default for FlushConfig {
60    fn default() -> Self {
61        Self {
62            // Physics/safety ceiling so linger defines segment size under load.
63            max_bytes: 1024 * 1024 * 1024,
64            max_batches: 10_000,
65            linger: Duration::from_millis(50),
66            // 1 = single-flight seal (best Local bulk); raise for parallel S3 PUTs.
67            max_inflight_flushes: 1,
68            max_buffered_bytes: 2 * 1024 * 1024 * 1024,
69            budget: BudgetConfig::default(),
70        }
71    }
72}
73
74/// Outcome of a [`LogEngine::produce`] call.
75#[derive(Clone, Debug, PartialEq, Eq)]
76pub struct AppendOutcome {
77    /// First assigned offset, when resolved at [`Durability::Sequenced`].
78    pub base_offset: Option<i64>,
79    /// Last assigned offset, when resolved at [`Durability::Sequenced`].
80    pub last_offset: Option<i64>,
81    /// Whether the batch is durably stored.
82    pub durable: bool,
83    /// Whether the batch has been sequenced (has an offset).
84    pub sequenced: bool,
85}
86
87/// A batch read back by [`LogEngine::fetch`], with its assigned base offset.
88#[derive(Clone, Debug, PartialEq, Eq)]
89pub struct FetchedBatch {
90    /// First offset of the batch.
91    pub base_offset: i64,
92    /// Number of records in the batch.
93    pub record_count: i32,
94    /// The opaque batch payload.
95    pub payload: Bytes,
96}
97
98/// Snapshot of the engine's buffering envelope.
99#[derive(Clone, Debug, PartialEq, Eq)]
100pub struct BufferStats {
101    /// Bytes still waiting in the mutable accumulation queue.
102    pub queued_bytes: usize,
103    /// Bytes owned by queued plus sealed in-flight flush work.
104    pub bytes_in_use: usize,
105    /// Batches still waiting in the mutable accumulation queue.
106    pub queued_batches: usize,
107    /// Configured upper bound for `bytes_in_use`.
108    pub max_buffered_bytes: usize,
109}
110
111type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
112
113fn retryable_storage_error(err: &ObjectLogError) -> bool {
114    matches!(err, ObjectLogError::StorageUnavailable(_))
115}
116
117async fn retry_delay(attempt: usize) {
118    let multiplier = 1u32 << attempt.min(5);
119    tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
120}
121
122async fn put_chunks_with_retries(
123    blob: &Arc<dyn BlobStore>,
124    key: &str,
125    chunks: Vec<Bytes>,
126) -> Result<(), ObjectLogError> {
127    // `Bytes::clone` is refcount-only; Local put_chunks streams without re-merge.
128    let mut attempt = 0usize;
129    loop {
130        match blob.put_chunks(key, chunks.clone()).await {
131            Ok(()) => return Ok(()),
132            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
133                retry_delay(attempt).await;
134                attempt += 1;
135            }
136            Err(err) => return Err(err),
137        }
138    }
139}
140
141async fn get_range_with_retries(
142    blob: &Arc<dyn BlobStore>,
143    key: &str,
144    range: std::ops::Range<u64>,
145) -> Result<Option<Bytes>, ObjectLogError> {
146    let mut attempt = 0usize;
147    loop {
148        match blob.get_range(key, range.clone()).await {
149            Ok(bytes) => return Ok(bytes),
150            Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
151                retry_delay(attempt).await;
152                attempt += 1;
153            }
154            Err(err) => return Err(err),
155        }
156    }
157}
158
159struct Pending<M> {
160    partition: PartitionKey,
161    record_count: i32,
162    payload: Bytes,
163    meta: M,
164    durability: Durability,
165    responder: Option<Responder>,
166    /// Monotonic enqueue id for [`LogEngine::flush`] barriers.
167    seq: u64,
168}
169
170struct FlushWork<M> {
171    batch: Vec<Pending<M>>,
172    locations: Vec<BatchLocation>,
173    responders: Vec<(Durability, Option<Responder>)>,
174    bytes: usize,
175    /// Highest enqueue seq included in this flush object.
176    max_seq: u64,
177    put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
178    put_started: Instant,
179    put_result: Option<Result<Duration, ObjectLogError>>,
180}
181
182enum TakeBatch<M> {
183    Batch(Vec<Pending<M>>),
184    Empty,
185    Shutdown,
186}
187
188type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
189
190struct Queue<M> {
191    items: VecDeque<Pending<M>>,
192    bytes: usize,
193    bytes_in_use: usize,
194    shutdown: bool,
195    /// Next seq to assign on enqueue.
196    next_seq: u64,
197    /// Highest seq fully sealed (put + sequencer commit finished).
198    completed_through: u64,
199    /// When true, take_batch flushes even if under size/linger.
200    force_flush: bool,
201    /// Waiters: (barrier_seq inclusive, responder).
202    flush_waiters: Vec<FlushWaiter>,
203    /// Last successful produce enqueue (for idle early-flush).
204    last_enqueue: Option<Instant>,
205    /// Enqueue time of the front item (linger deadline = oldest + linger).
206    oldest_enqueue: Option<Instant>,
207}
208
209struct Shared<M> {
210    queue: Mutex<Queue<M>>,
211    cv: Condvar,
212    max_buffered_bytes: usize,
213    /// Budget controller + inspectable counters (TD-004).
214    budget: Mutex<BudgetRuntime>,
215    /// Max linger (operator ceiling) and budget config copy for the flush loop.
216    flush_config: FlushConfig,
217}
218
219/// A buffered, multiplexing append-log engine over a [`BlobStore`], with
220/// sequencing delegated to a [`Sequencer`].
221///
222/// `produce` group-commits: many batches across many partitions are multiplexed
223/// into one object, PUT durably, then handed to the sequencer in a single call —
224/// so PUT count is decoupled from produce count. A single flush worker preserves
225/// per-[`PartitionKey`] arrival order and never splits a partition across
226/// concurrent commits.
227pub struct LogEngine<S: Sequencer> {
228    shared: Arc<Shared<S::Meta>>,
229    blob: Arc<dyn BlobStore>,
230    sequencer: Arc<S>,
231    flush_thread: Option<JoinHandle<()>>,
232}
233
234impl<S> LogEngine<S>
235where
236    S: Sequencer + 'static,
237    S::Meta: Send + 'static,
238{
239    /// Create an engine over `blob` and `sequencer` with the given flush policy.
240    /// Objects are keyed `<key_prefix><counter>`; pick a prefix unique to this
241    /// engine instance if several share a store.
242    pub fn new(
243        blob: Arc<dyn BlobStore>,
244        sequencer: Arc<S>,
245        config: FlushConfig,
246        key_prefix: impl Into<String>,
247    ) -> Self {
248        if let Err(msg) = config.budget.validate() {
249            panic!("invalid FlushConfig.budget: {msg}");
250        }
251        if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
252            eprintln!("object-log flush config: {config:?}");
253        }
254        let budget_rt = BudgetRuntime::new(config.budget);
255        let shared = Arc::new(Shared {
256            queue: Mutex::new(Queue {
257                items: VecDeque::new(),
258                bytes: 0,
259                bytes_in_use: 0,
260                shutdown: false,
261                next_seq: 1, // first enqueued batch gets seq 1
262                completed_through: 0,
263                force_flush: false,
264                flush_waiters: Vec::new(),
265                last_enqueue: None,
266                oldest_enqueue: None,
267            }),
268            cv: Condvar::new(),
269            max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
270            budget: Mutex::new(budget_rt),
271            flush_config: config,
272        });
273        let flush_thread = {
274            let shared = Arc::clone(&shared);
275            let blob = Arc::clone(&blob);
276            let sequencer = Arc::clone(&sequencer);
277            let prefix = key_prefix.into();
278            std::thread::Builder::new()
279                .name("object-log-flush".into())
280                .spawn(move || flush_loop(shared, blob, sequencer, config, prefix))
281                .expect("spawn flush thread")
282        };
283        Self {
284            shared,
285            blob,
286            sequencer,
287            flush_thread: Some(flush_thread),
288        }
289    }
290
291    /// Buffer a batch and resolve at the requested [`Durability`].
292    pub async fn produce(
293        &self,
294        partition: PartitionKey,
295        payload: Bytes,
296        record_count: i32,
297        meta: S::Meta,
298        durability: Durability,
299    ) -> Result<AppendOutcome, ObjectLogError> {
300        if record_count <= 0 {
301            return Err(ObjectLogError::InvalidBatch(
302                "record_count must be > 0".into(),
303            ));
304        }
305        if matches!(durability, Durability::Buffered) {
306            self.enqueue(Pending {
307                partition,
308                record_count,
309                payload,
310                meta,
311                durability,
312                responder: None,
313                seq: 0, // filled in enqueue
314            })?;
315            return Ok(AppendOutcome {
316                base_offset: None,
317                last_offset: None,
318                durable: false,
319                sequenced: false,
320            });
321        }
322        // fail_closed: reserve predicted media ops before waiting on flush.
323        {
324            let mut budget = self.shared.budget.lock().expect("poisoned");
325            if budget.config.enabled
326                && budget.config.mode == BudgetMode::FailClosed
327                && !budget.reserve_for_fail_closed(Instant::now())
328            {
329                return Err(ObjectLogError::BudgetExceeded(
330                    "insufficient durable-ops budget to admit produce".into(),
331                ));
332            }
333        }
334        // budget_priority: wait briefly for tokens when empty.
335        if self.shared.flush_config.budget.enabled
336            && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
337        {
338            let timeout = self.shared.flush_config.budget.admission_timeout;
339            let deadline = Instant::now() + timeout;
340            loop {
341                let mut budget = self.shared.budget.lock().expect("poisoned");
342                if budget.can_admit_now(Instant::now()) {
343                    break;
344                }
345                drop(budget);
346                if Instant::now() >= deadline {
347                    return Err(ObjectLogError::BudgetExceeded(
348                        "timed out waiting for durable-ops budget".into(),
349                    ));
350                }
351                std::thread::sleep(Duration::from_millis(1));
352            }
353        }
354        let (tx, rx) = oneshot::channel();
355        self.enqueue(Pending {
356            partition,
357            record_count,
358            payload,
359            meta,
360            durability,
361            responder: Some(tx),
362            seq: 0,
363        })?;
364        rx.await
365            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
366    }
367
368    /// Seal every batch enqueued **at or before** this call (barrier).
369    ///
370    /// Use after [`Durability::Buffered`] produces to wait for durable PUT (and
371    /// sequencing) of that prior work. Concurrent produces enqueued after this
372    /// call are not required to finish. Empty buffer returns immediately.
373    pub async fn flush(&self) -> Result<(), ObjectLogError> {
374        let rx = {
375            let mut q = self.shared.queue.lock().expect("poisoned");
376            if q.shutdown {
377                return Err(ObjectLogError::Sequencer("engine shutting down".into()));
378            }
379            if q.next_seq == 1 {
380                // Nothing has ever been enqueued.
381                return Ok(());
382            }
383            let barrier = q.next_seq - 1;
384            if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
385                return Ok(());
386            }
387            let (tx, rx) = oneshot::channel();
388            q.force_flush = true;
389            q.flush_waiters.push((barrier, tx));
390            self.shared.cv.notify_all();
391            rx
392        };
393        rx.await
394            .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
395    }
396
397    fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
398        let item_len = item.payload.len();
399        let max_buffered_bytes = self.shared.max_buffered_bytes;
400        let shared = Arc::clone(&self.shared);
401        let mut q = shared.queue.lock().expect("poisoned");
402        while !q.shutdown
403            && q.bytes_in_use > 0
404            && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
405        {
406            q = shared.cv.wait(q).expect("poisoned");
407        }
408        if q.shutdown {
409            return Err(ObjectLogError::Sequencer("engine shutting down".into()));
410        }
411        let now = Instant::now();
412        item.seq = q.next_seq;
413        q.next_seq = q.next_seq.saturating_add(1);
414        q.bytes += item_len;
415        q.bytes_in_use += item_len;
416        q.last_enqueue = Some(now);
417        if q.oldest_enqueue.is_none() {
418            q.oldest_enqueue = Some(now);
419        }
420        q.items.push_back(item);
421        shared.cv.notify_all();
422        Ok(())
423    }
424
425    /// Read batches covering offsets at/after `offset`, up to ~`max_bytes`.
426    pub async fn fetch(
427        &self,
428        partition: &PartitionKey,
429        offset: i64,
430        max_bytes: usize,
431    ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
432        let entries = self.sequencer.lookup(partition, offset)?;
433        let mut out = Vec::new();
434        let mut total = 0usize;
435        for e in entries {
436            if total >= max_bytes && !out.is_empty() {
437                break;
438            }
439            let start = e.location.byte_start as u64;
440            let end = start + e.location.byte_len as u64;
441            let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
442                .await?
443                .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
444            total += bytes.len();
445            out.push(FetchedBatch {
446                base_offset: e.base_offset,
447                record_count: e.record_count,
448                payload: bytes,
449            });
450        }
451        Ok(out)
452    }
453
454    /// Drop the partition's log below `offset` and delete any object that thereby
455    /// becomes fully unreferenced.
456    pub async fn truncate_before(
457        &self,
458        partition: &PartitionKey,
459        offset: i64,
460    ) -> Result<(), ObjectLogError> {
461        let dead = self.sequencer.truncate_before(partition, offset)?;
462        for object_id in dead {
463            self.blob.delete(&object_id).await?;
464        }
465        Ok(())
466    }
467
468    /// Return a point-in-time snapshot of queued and in-flight payload bytes.
469    pub fn buffer_stats(&self) -> BufferStats {
470        let q = self.shared.queue.lock().expect("poisoned");
471        BufferStats {
472            queued_bytes: q.bytes,
473            bytes_in_use: q.bytes_in_use,
474            queued_batches: q.items.len(),
475            max_buffered_bytes: self.shared.max_buffered_bytes,
476        }
477    }
478
479    /// Inspect budget controller layers and counters (TD-004).
480    pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
481        let now = Instant::now();
482        let mut budget = self.shared.budget.lock().expect("poisoned");
483        budget.refill(now);
484        let effective = budget.effective_budget_per_sec;
485        let reason = if !budget.config.enabled {
486            EffectiveReason::Disabled
487        } else if budget.config.budget_per_sec_cap.is_some()
488            && budget
489                .config
490                .budget_per_sec_cap
491                .is_some_and(|c| (effective - c).abs() < 1e-9)
492        {
493            EffectiveReason::ConfigCap
494        } else if budget.ongoing_capacity.is_some() {
495            EffectiveReason::Ongoing
496        } else if budget.startup_capacity.is_some() {
497            EffectiveReason::StartupProbe
498        } else {
499            EffectiveReason::DefaultCapacity
500        };
501        let (queued, last_enq) = {
502            let q = self.shared.queue.lock().expect("poisoned");
503            (q.bytes, q.last_enqueue)
504        };
505        let early = budget.allow_early_flush(now, queued, last_enq);
506        let max_linger = self.shared.flush_config.linger;
507        let eff_linger = if !budget.config.enabled {
508            max_linger
509        } else if early || max_linger.is_zero() {
510            Duration::ZERO
511        } else {
512            max_linger
513        };
514        PipelineSnapshot {
515            budget_per_sec: EffectiveKnob {
516                configured: budget.config.budget_per_sec_cap,
517                startup_measured: budget.startup_capacity,
518                ongoing_measured: budget.ongoing_capacity,
519                effective,
520                reason,
521            },
522            effective_linger_ms: EffectiveKnob {
523                configured: Some(max_linger.as_millis() as u64),
524                startup_measured: None,
525                ongoing_measured: None,
526                effective: eff_linger.as_millis() as u64,
527                reason: if early {
528                    EffectiveReason::Ongoing
529                } else {
530                    EffectiveReason::Configured
531                },
532            },
533            max_linger_ms: max_linger.as_millis() as u64,
534            token_fill_ratio: budget.fill_ratio(),
535            media_ops_total: budget.media_ops_total,
536            overdraft_total: budget.overdraft_total,
537            flushes_total: budget.flushes_total,
538            undersized_deadline_flushes: budget.undersized_deadline_flushes,
539            predicted_media_ops: budget.predicted_media_ops,
540            budget_enabled: budget.config.enabled,
541            budget_mode: budget.config.mode,
542        }
543    }
544}
545
546impl<S: Sequencer> Drop for LogEngine<S> {
547    fn drop(&mut self) {
548        {
549            let mut q = self.shared.queue.lock().expect("poisoned");
550            q.shutdown = true;
551        }
552        self.shared.cv.notify_all();
553        if let Some(t) = self.flush_thread.take() {
554            let _ = t.join();
555        }
556    }
557}
558
559fn flush_loop<S>(
560    shared: Arc<Shared<S::Meta>>,
561    blob: Arc<dyn BlobStore>,
562    sequencer: Arc<S>,
563    config: FlushConfig,
564    prefix: String,
565) where
566    S: Sequencer + 'static,
567    S::Meta: Send + 'static,
568{
569    let max_inflight = config.max_inflight_flushes.max(1);
570    let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
571        .ok()
572        .and_then(|v| v.parse::<usize>().ok())
573        .filter(|v| *v > 0)
574        .unwrap_or_else(|| max_inflight.min(8));
575    let rt = tokio::runtime::Builder::new_multi_thread()
576        .worker_threads(worker_threads)
577        .enable_all()
578        .build()
579        .expect("flush runtime");
580    let mut counter: u64 = 0;
581    let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
582    let mut active_puts = 0usize;
583    let mut shutdown = false;
584
585    loop {
586        while !shutdown && active_puts < max_inflight {
587            let wait_for_more = pending.is_empty();
588            match take_batch(&shared, config, wait_for_more) {
589                TakeBatch::Batch(batch) => {
590                    counter += 1;
591                    let concurrent = max_inflight > 1;
592                    let work = start_flush_work(&rt, &blob, &prefix, counter, batch, concurrent);
593                    if concurrent {
594                        active_puts += 1;
595                    }
596                    pending.push_back(work);
597                }
598                TakeBatch::Empty => break,
599                TakeBatch::Shutdown => shutdown = true,
600            }
601        }
602
603        let mut made_progress = false;
604        for work in pending.iter_mut() {
605            let Some(put) = work.put.as_ref() else {
606                continue;
607            };
608            if !put.is_finished() {
609                continue;
610            }
611            let put = work.put.take().expect("put handle exists");
612            let elapsed = work.put_started.elapsed();
613            work.put_result = Some(match rt.block_on(put) {
614                Ok(Ok(())) => Ok(elapsed),
615                Ok(Err(e)) => Err(e),
616                Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
617                    "flush task failed: {e}"
618                ))),
619            });
620            active_puts = active_puts.saturating_sub(1);
621            made_progress = true;
622        }
623
624        while pending
625            .front()
626            .is_some_and(|work| work.put_result.is_some())
627        {
628            let work = pending.pop_front().expect("front exists");
629            let released = finish_flush_work(&shared, &blob, &sequencer, work);
630            let mut q = shared.queue.lock().expect("poisoned");
631            q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
632            shared.cv.notify_all();
633            made_progress = true;
634        }
635
636        if pending.is_empty() {
637            {
638                let mut q = shared.queue.lock().expect("poisoned");
639                if q.items.is_empty() && q.bytes_in_use == 0 {
640                    // Nothing in flight: any remaining waiters with barrier already
641                    // covered (or no work after last seal) can complete.
642                    notify_flush_waiters(&mut q);
643                }
644            }
645            shared.cv.notify_all();
646            if shutdown {
647                // Fail any remaining flush waiters.
648                let mut q = shared.queue.lock().expect("poisoned");
649                for (_, tx) in q.flush_waiters.drain(..) {
650                    let _ = tx.send(Err(ObjectLogError::Sequencer(
651                        "engine shutting down".into(),
652                    )));
653                }
654                return;
655            }
656            continue;
657        }
658
659        if !made_progress {
660            std::thread::sleep(Duration::from_millis(1));
661        }
662    }
663}
664
665/// How long to sleep before re-evaluating a seal decision.
666/// - `ZERO` means seal now (early-flush or linger deadline hit).
667/// - Otherwise wait at most this long (interrupted by new produces).
668fn effective_linger<M>(
669    shared: &Shared<M>,
670    config: FlushConfig,
671    queued_bytes: usize,
672    last_enqueue: Option<Instant>,
673    oldest_enqueue: Option<Instant>,
674) -> Duration {
675    if config.linger.is_zero() {
676        return Duration::ZERO;
677    }
678    let now = Instant::now();
679    if config.budget.enabled {
680        let mut budget = shared.budget.lock().expect("poisoned");
681        budget.refill(now);
682        if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
683            budget.note_early_flush(now);
684            return Duration::ZERO;
685        }
686    }
687    // True linger: seal when the oldest buffered item reaches max wait.
688    if let Some(oldest) = oldest_enqueue {
689        let deadline = oldest + config.linger;
690        if now >= deadline {
691            return Duration::ZERO;
692        }
693        let until_deadline = deadline.saturating_duration_since(now);
694        // Also wake when idle gate could open, so sparse produces don't wait full linger.
695        if config.budget.enabled
696            && !config.budget.early_flush_idle.is_zero()
697            && let Some(last) = last_enqueue
698        {
699            let idle_at = last + config.budget.early_flush_idle;
700            if idle_at > now && idle_at < deadline {
701                return idle_at.saturating_duration_since(now);
702            }
703        }
704        return until_deadline;
705    }
706    config.linger
707}
708
709fn take_batch<M>(
710    shared: &Arc<Shared<M>>,
711    config: FlushConfig,
712    wait_when_empty: bool,
713) -> TakeBatch<M> {
714    let mut q = shared.queue.lock().expect("poisoned");
715    loop {
716        if q.items.is_empty() {
717            if q.shutdown {
718                return TakeBatch::Shutdown;
719            }
720            if !wait_when_empty {
721                return TakeBatch::Empty;
722            }
723            let linger = {
724                let queued = q.bytes;
725                let last_enq = q.last_enqueue;
726                let oldest = q.oldest_enqueue;
727                drop(q);
728                let l = effective_linger(shared, config, queued, last_enq, oldest);
729                q = shared.queue.lock().expect("poisoned");
730                l
731            };
732            if linger.is_zero() {
733                q = shared.cv.wait(q).expect("poisoned");
734            } else {
735                let (guard, timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
736                q = guard;
737                if timeout.timed_out() && q.items.is_empty() {
738                    return TakeBatch::Empty;
739                }
740            }
741            continue;
742        }
743
744        let linger = {
745            let queued = q.bytes;
746            let last_enq = q.last_enqueue;
747            let oldest = q.oldest_enqueue;
748            drop(q);
749            let l = effective_linger(shared, config, queued, last_enq, oldest);
750            q = shared.queue.lock().expect("poisoned");
751            l
752        };
753
754        let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
755        let force = q.force_flush;
756        let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
757        if triggered {
758            if !size_trigger && !q.shutdown && !q.items.is_empty() {
759                let mut budget = shared.budget.lock().expect("poisoned");
760                if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
761                    budget.undersized_deadline_flushes += 1;
762                }
763            }
764            break;
765        }
766
767        // Short wait (until idle gate or linger deadline); re-evaluate, do not
768        // seal solely because a probe sleep timed out.
769        let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
770        q = guard;
771    }
772
773    let force_drain = q.force_flush;
774    let mut items = Vec::new();
775    let mut bytes = 0usize;
776    // On force_flush, still respect max_bytes so one object stays bounded, but
777    // ignore max_batches so a barrier can drain large queues across objects.
778    while let Some(item) = q.items.pop_front() {
779        q.bytes = q.bytes.saturating_sub(item.payload.len());
780        bytes += item.payload.len();
781        items.push(item);
782        if bytes >= config.max_bytes {
783            break;
784        }
785        if !force_drain && items.len() >= config.max_batches {
786            break;
787        }
788    }
789    q.oldest_enqueue = if q.items.is_empty() {
790        None
791    } else {
792        // Approximate: next seal window starts now for remaining items.
793        Some(Instant::now())
794    };
795    // Keep forcing while a barrier still has queued work.
796    if force_drain {
797        q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
798    }
799    if items.is_empty() {
800        TakeBatch::Empty
801    } else {
802        TakeBatch::Batch(items)
803    }
804}
805
806fn notify_flush_waiters<M>(q: &mut Queue<M>) {
807    let done = q.completed_through;
808    let mut i = 0;
809    while i < q.flush_waiters.len() {
810        if q.flush_waiters[i].0 <= done {
811            let (_, tx) = q.flush_waiters.swap_remove(i);
812            let _ = tx.send(Ok(()));
813        } else {
814            i += 1;
815        }
816    }
817    if q.flush_waiters.is_empty() {
818        q.force_flush = false;
819    } else if q.items.is_empty() && q.bytes_in_use == 0 {
820        // Barriers still waiting but no work — should not happen; clear force.
821        q.force_flush = true;
822    } else {
823        q.force_flush = true;
824    }
825}
826
827fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
828    for (_, tx) in responders.iter_mut() {
829        if let Some(tx) = tx.take() {
830            let _ = tx.send(Err(err.clone()));
831        }
832    }
833}
834
835fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
836    for (durability, tx) in responders.iter_mut() {
837        if *durability == Durability::Durable
838            && let Some(tx) = tx.take()
839        {
840            let _ = tx.send(Ok(AppendOutcome {
841                base_offset: None,
842                last_offset: None,
843                durable: true,
844                sequenced: false,
845            }));
846        }
847    }
848}
849
850fn prepare_flush_work<M>(
851    prefix: &str,
852    counter: u64,
853    mut batch: Vec<Pending<M>>,
854) -> (FlushWork<M>, String, Vec<Bytes>) {
855    let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
856    let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
857    let key = format!("{prefix}{counter:020}");
858    let mut offset = 0usize;
859    for p in &batch {
860        let start = offset as u32;
861        offset += p.payload.len();
862        chunks.push(p.payload.clone());
863        locations.push(BatchLocation {
864            object_id: key.clone(),
865            byte_start: start,
866            byte_len: p.payload.len() as u32,
867        });
868    }
869    if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
870        eprintln!(
871            "object-log seal: key={key} batches={} bytes={offset}",
872            batch.len()
873        );
874    }
875    let responders: Vec<(Durability, Option<Responder>)> = batch
876        .iter_mut()
877        .map(|p| (p.durability, p.responder.take()))
878        .collect();
879    let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
880    let work = FlushWork {
881        batch,
882        locations,
883        responders,
884        bytes: offset,
885        max_seq,
886        put: None,
887        put_started: Instant::now(),
888        put_result: None,
889    };
890    (work, key, chunks)
891}
892
893/// Start a durable put for a sealed batch.
894///
895/// - `concurrent == false` (default `max_inflight_flushes == 1`): `block_on` the
896///   put on the flush thread; Local uses `block_in_place` (no spawn_blocking queue).
897/// - `concurrent == true`: spawn put tasks for parallel S3-style throughput.
898fn start_flush_work<M>(
899    rt: &tokio::runtime::Runtime,
900    blob: &Arc<dyn BlobStore>,
901    prefix: &str,
902    counter: u64,
903    batch: Vec<Pending<M>>,
904    concurrent: bool,
905) -> FlushWork<M> {
906    let (mut work, key, chunks) = prepare_flush_work(prefix, counter, batch);
907    let blob = Arc::clone(blob);
908    let _ = blob.take_media_op_stats();
909    work.put_started = Instant::now();
910    if concurrent {
911        work.put =
912            Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
913    } else {
914        let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
915        let elapsed = work.put_started.elapsed();
916        work.put_result = Some(match put_result {
917            Ok(()) => Ok(elapsed),
918            Err(e) => Err(e),
919        });
920    }
921    work
922}
923
924fn finish_flush_work<S, M>(
925    shared: &Arc<Shared<M>>,
926    blob: &Arc<dyn BlobStore>,
927    sequencer: &Arc<S>,
928    mut work: FlushWork<S::Meta>,
929) -> usize
930where
931    S: Sequencer + 'static,
932    S::Meta: Send + 'static,
933{
934    let release_bytes = work.bytes;
935    // Durable-then-sequence: the object PUT may have overlapped later PUTs, but
936    // sequencer commits are still completed in object creation order.
937    let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
938    let put_elapsed = match work.put_result.take().expect("put result is ready") {
939        Ok(elapsed) => elapsed,
940        Err(e) => {
941            send_storage_error(&mut work.responders, e.clone());
942            let mut q = shared.queue.lock().expect("poisoned");
943            // Unblock flush waiters covering this seq with the storage error.
944            let done = work.max_seq;
945            if done >= q.completed_through {
946                q.completed_through = done;
947            }
948            let mut i = 0;
949            while i < q.flush_waiters.len() {
950                if q.flush_waiters[i].0 <= done {
951                    let (_, tx) = q.flush_waiters.swap_remove(i);
952                    let _ = tx.send(Err(e.clone()));
953                } else {
954                    i += 1;
955                }
956            }
957            shared.cv.notify_all();
958            return release_bytes;
959        }
960    };
961
962    // Media ops for the data object put.
963    let mut media_ops = blob.take_media_op_stats().map(|s| s.media_ops).unwrap_or(1); // fallback: 1 per successful put
964
965    // Signal Durable-level waiters now (after PUT, before commit).
966    send_durable_acks(&mut work.responders);
967
968    // Sequence the whole object atomically.
969    let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
970        .batch
971        .iter()
972        .zip(work.locations.iter())
973        .map(|(p, loc)| CommitBatch {
974            partition: p.partition.clone(),
975            record_count: p.record_count,
976            location: loc.clone(),
977            meta: &p.meta,
978        })
979        .collect();
980
981    let commit_started = timing.then(Instant::now);
982    // Clear stats again so sequencer durable work (e.g. ManifestSequencer put)
983    // on the shared store is counted into the same budget.
984    let _ = blob.take_media_op_stats();
985    match sequencer.commit(&commit_batches) {
986        Ok(outcomes) => {
987            if let Some(commit_started) = commit_started {
988                eprintln!(
989                    "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={}",
990                    work.bytes,
991                    commit_batches.len(),
992                    put_elapsed.as_millis(),
993                    commit_started.elapsed().as_millis()
994                );
995            }
996            if let Some(s) = blob.take_media_op_stats() {
997                media_ops = media_ops.saturating_add(s.media_ops);
998            }
999            for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1000                if let Some(tx) = tx.take() {
1001                    let (base, last) = match outcome {
1002                        CommitOutcome::Assigned {
1003                            base_offset,
1004                            record_count,
1005                        } => (
1006                            Some(base_offset),
1007                            Some(base_offset + record_count as i64 - 1),
1008                        ),
1009                        CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1010                    };
1011                    let _ = tx.send(Ok(AppendOutcome {
1012                        base_offset: base,
1013                        last_offset: last,
1014                        durable: true,
1015                        sequenced: true,
1016                    }));
1017                }
1018            }
1019        }
1020        Err(e) => {
1021            if let Some(s) = blob.take_media_op_stats() {
1022                media_ops = media_ops.saturating_add(s.media_ops);
1023            }
1024            send_storage_error(&mut work.responders, e);
1025        }
1026    }
1027
1028    {
1029        let mut budget = shared.budget.lock().expect("poisoned");
1030        budget.consume_after_flush(media_ops, Instant::now());
1031    }
1032    {
1033        let mut q = shared.queue.lock().expect("poisoned");
1034        if work.max_seq >= q.completed_through {
1035            q.completed_through = work.max_seq;
1036        }
1037        notify_flush_waiters(&mut q);
1038    }
1039    shared.cv.notify_all();
1040
1041    release_bytes
1042}