1use crate::budget::{
4 BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24 Buffered,
27 Durable,
30 Sequenced,
33}
34
35#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38 pub max_bytes: usize,
43 pub max_batches: usize,
45 pub linger: Duration,
51 pub max_inflight_flushes: usize,
54 pub max_buffered_bytes: usize,
57 pub budget: BudgetConfig,
59}
60
61impl Default for FlushConfig {
62 fn default() -> Self {
63 Self {
64 max_bytes: 1024 * 1024 * 1024,
66 max_batches: 10_000,
67 linger: Duration::from_millis(50),
68 max_inflight_flushes: 1,
70 max_buffered_bytes: 2 * 1024 * 1024 * 1024,
71 budget: BudgetConfig::default(),
72 }
73 }
74}
75
76#[derive(Clone, Debug, PartialEq, Eq)]
78pub struct AppendOutcome {
79 pub base_offset: Option<i64>,
81 pub last_offset: Option<i64>,
83 pub durable: bool,
85 pub sequenced: bool,
87}
88
89#[derive(Clone, Debug, PartialEq, Eq)]
91pub struct FetchedBatch {
92 pub base_offset: i64,
94 pub record_count: i32,
96 pub payload: Bytes,
98}
99
100#[derive(Clone, Debug, PartialEq, Eq)]
102pub struct BufferStats {
103 pub queued_bytes: usize,
105 pub bytes_in_use: usize,
107 pub queued_batches: usize,
109 pub max_buffered_bytes: usize,
111}
112
113type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
114
115fn retryable_storage_error(err: &ObjectLogError) -> bool {
116 matches!(err, ObjectLogError::StorageUnavailable(_))
117}
118
119async fn retry_delay(attempt: usize) {
120 let multiplier = 1u32 << attempt.min(5);
121 tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
122}
123
124async fn put_chunks_with_retries(
125 blob: &Arc<dyn BlobStore>,
126 key: &str,
127 chunks: Vec<Bytes>,
128) -> Result<(), ObjectLogError> {
129 let mut attempt = 0usize;
131 loop {
132 match blob.put_chunks(key, chunks.clone()).await {
133 Ok(()) => return Ok(()),
134 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
135 retry_delay(attempt).await;
136 attempt += 1;
137 }
138 Err(err) => return Err(err),
139 }
140 }
141}
142
143async fn get_range_with_retries(
144 blob: &Arc<dyn BlobStore>,
145 key: &str,
146 range: std::ops::Range<u64>,
147) -> Result<Option<Bytes>, ObjectLogError> {
148 let mut attempt = 0usize;
149 loop {
150 match blob.get_range(key, range.clone()).await {
151 Ok(bytes) => return Ok(bytes),
152 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
153 retry_delay(attempt).await;
154 attempt += 1;
155 }
156 Err(err) => return Err(err),
157 }
158 }
159}
160
161struct Pending<M> {
162 partition: PartitionKey,
163 record_count: i32,
164 payload: Bytes,
165 meta: M,
166 durability: Durability,
167 responder: Option<Responder>,
168 seq: u64,
170}
171
172struct FlushWork<M> {
173 batch: Vec<Pending<M>>,
174 locations: Vec<BatchLocation>,
175 responders: Vec<(Durability, Option<Responder>)>,
176 bytes: usize,
177 max_seq: u64,
179 put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
180 put_started: Instant,
181 put_result: Option<Result<Duration, ObjectLogError>>,
182}
183
184enum TakeBatch<M> {
185 Batch(Vec<Pending<M>>),
186 Empty,
187 Shutdown,
188}
189
190type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
191
192struct Queue<M> {
193 items: VecDeque<Pending<M>>,
194 bytes: usize,
195 bytes_in_use: usize,
196 shutdown: bool,
197 next_seq: u64,
199 completed_through: u64,
201 force_flush: bool,
203 flush_waiters: Vec<FlushWaiter>,
205 last_enqueue: Option<Instant>,
207 oldest_enqueue: Option<Instant>,
209}
210
211struct Shared<M> {
212 queue: Mutex<Queue<M>>,
213 cv: Condvar,
214 max_buffered_bytes: usize,
215 budget: Mutex<BudgetRuntime>,
217 flush_config: FlushConfig,
219}
220
221pub struct LogEngine<S: Sequencer> {
230 shared: Arc<Shared<S::Meta>>,
231 blob: Arc<dyn BlobStore>,
232 sequencer: Arc<S>,
233 key_prefix: String,
235 flush_thread: Option<JoinHandle<()>>,
236}
237
238impl<S> LogEngine<S>
239where
240 S: Sequencer + 'static,
241 S::Meta: Send + 'static,
242{
243 pub fn new(
247 blob: Arc<dyn BlobStore>,
248 sequencer: Arc<S>,
249 config: FlushConfig,
250 key_prefix: impl Into<String>,
251 ) -> Self {
252 if let Err(msg) = config.budget.validate() {
253 panic!("invalid FlushConfig.budget: {msg}");
254 }
255 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
256 eprintln!("object-log flush config: {config:?}");
257 }
258 let budget_rt = BudgetRuntime::new(config.budget);
259 let shared = Arc::new(Shared {
260 queue: Mutex::new(Queue {
261 items: VecDeque::new(),
262 bytes: 0,
263 bytes_in_use: 0,
264 shutdown: false,
265 next_seq: 1, completed_through: 0,
267 force_flush: false,
268 flush_waiters: Vec::new(),
269 last_enqueue: None,
270 oldest_enqueue: None,
271 }),
272 cv: Condvar::new(),
273 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
274 budget: Mutex::new(budget_rt),
275 flush_config: config,
276 });
277 let key_prefix = key_prefix.into();
278 let flush_thread = {
279 let shared = Arc::clone(&shared);
280 let blob = Arc::clone(&blob);
281 let sequencer = Arc::clone(&sequencer);
282 let prefix = key_prefix.clone();
283 std::thread::Builder::new()
284 .name("object-log-flush".into())
285 .spawn(move || flush_loop(shared, blob, sequencer, config, prefix))
286 .expect("spawn flush thread")
287 };
288 Self {
289 shared,
290 blob,
291 sequencer,
292 key_prefix,
293 flush_thread: Some(flush_thread),
294 }
295 }
296
297 pub fn data_prefix(&self) -> &str {
299 &self.key_prefix
300 }
301
302 pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
304 &self.blob
305 }
306
307 pub fn sequencer(&self) -> &Arc<S> {
309 &self.sequencer
310 }
311
312 pub async fn produce(
314 &self,
315 partition: PartitionKey,
316 payload: Bytes,
317 record_count: i32,
318 meta: S::Meta,
319 durability: Durability,
320 ) -> Result<AppendOutcome, ObjectLogError> {
321 if record_count <= 0 {
322 return Err(ObjectLogError::InvalidBatch(
323 "record_count must be > 0".into(),
324 ));
325 }
326 if matches!(durability, Durability::Buffered) {
327 self.enqueue(Pending {
328 partition,
329 record_count,
330 payload,
331 meta,
332 durability,
333 responder: None,
334 seq: 0, })?;
336 return Ok(AppendOutcome {
337 base_offset: None,
338 last_offset: None,
339 durable: false,
340 sequenced: false,
341 });
342 }
343 {
345 let mut budget = self.shared.budget.lock().expect("poisoned");
346 if budget.config.enabled
347 && budget.config.mode == BudgetMode::FailClosed
348 && !budget.reserve_for_fail_closed(Instant::now())
349 {
350 return Err(ObjectLogError::BudgetExceeded(
351 "insufficient durable-ops budget to admit produce".into(),
352 ));
353 }
354 }
355 if self.shared.flush_config.budget.enabled
357 && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
358 {
359 let timeout = self.shared.flush_config.budget.admission_timeout;
360 let deadline = Instant::now() + timeout;
361 loop {
362 let mut budget = self.shared.budget.lock().expect("poisoned");
363 if budget.can_admit_now(Instant::now()) {
364 break;
365 }
366 drop(budget);
367 if Instant::now() >= deadline {
368 return Err(ObjectLogError::BudgetExceeded(
369 "timed out waiting for durable-ops budget".into(),
370 ));
371 }
372 std::thread::sleep(Duration::from_millis(1));
373 }
374 }
375 let (tx, rx) = oneshot::channel();
376 self.enqueue(Pending {
377 partition,
378 record_count,
379 payload,
380 meta,
381 durability,
382 responder: Some(tx),
383 seq: 0,
384 })?;
385 rx.await
386 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
387 }
388
389 pub async fn flush(&self) -> Result<(), ObjectLogError> {
395 let rx = {
396 let mut q = self.shared.queue.lock().expect("poisoned");
397 if q.shutdown {
398 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
399 }
400 if q.next_seq == 1 {
401 return Ok(());
403 }
404 let barrier = q.next_seq - 1;
405 if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
406 return Ok(());
407 }
408 let (tx, rx) = oneshot::channel();
409 q.force_flush = true;
410 q.flush_waiters.push((barrier, tx));
411 self.shared.cv.notify_all();
412 rx
413 };
414 rx.await
415 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
416 }
417
418 fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
419 let item_len = item.payload.len();
420 let max_buffered_bytes = self.shared.max_buffered_bytes;
421 let shared = Arc::clone(&self.shared);
422 let mut q = shared.queue.lock().expect("poisoned");
423 while !q.shutdown
424 && q.bytes_in_use > 0
425 && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
426 {
427 q = shared.cv.wait(q).expect("poisoned");
428 }
429 if q.shutdown {
430 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
431 }
432 let now = Instant::now();
433 item.seq = q.next_seq;
434 q.next_seq = q.next_seq.saturating_add(1);
435 q.bytes += item_len;
436 q.bytes_in_use += item_len;
437 q.last_enqueue = Some(now);
438 if q.oldest_enqueue.is_none() {
439 q.oldest_enqueue = Some(now);
440 }
441 q.items.push_back(item);
442 shared.cv.notify_all();
443 Ok(())
444 }
445
446 pub async fn fetch(
448 &self,
449 partition: &PartitionKey,
450 offset: i64,
451 max_bytes: usize,
452 ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
453 let entries = self.sequencer.lookup(partition, offset)?;
454 let mut out = Vec::new();
455 let mut total = 0usize;
456 for e in entries {
457 if total >= max_bytes && !out.is_empty() {
458 break;
459 }
460 let batch = self.load_entry(&e).await?;
461 total += batch.payload.len();
462 out.push(batch);
463 }
464 Ok(out)
465 }
466
467 pub async fn fetch_stream<F>(
474 &self,
475 partition: &PartitionKey,
476 offset: i64,
477 mut visit: F,
478 ) -> Result<(), ObjectLogError>
479 where
480 F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
481 {
482 let entries = self.sequencer.lookup(partition, offset)?;
483 for e in entries {
484 visit(self.load_entry(&e).await?)?;
485 }
486 Ok(())
487 }
488
489 async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
490 let start = e.location.byte_start as u64;
491 let end = start + e.location.byte_len as u64;
492 let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
493 .await?
494 .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
495 Ok(FetchedBatch {
496 base_offset: e.base_offset,
497 record_count: e.record_count,
498 payload: bytes,
499 })
500 }
501
502 pub async fn truncate_before(
505 &self,
506 partition: &PartitionKey,
507 offset: i64,
508 ) -> Result<(), ObjectLogError> {
509 let dead = self.sequencer.truncate_before(partition, offset)?;
510 for object_id in dead {
511 self.blob.delete(&object_id).await?;
512 }
513 Ok(())
514 }
515
516 pub async fn reap_orphans(
526 &self,
527 live: &HashSet<String>,
528 ) -> Result<Vec<String>, ObjectLogError> {
529 reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
530 }
531
532 pub fn buffer_stats(&self) -> BufferStats {
534 let q = self.shared.queue.lock().expect("poisoned");
535 BufferStats {
536 queued_bytes: q.bytes,
537 bytes_in_use: q.bytes_in_use,
538 queued_batches: q.items.len(),
539 max_buffered_bytes: self.shared.max_buffered_bytes,
540 }
541 }
542
543 pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
545 let now = Instant::now();
546 let mut budget = self.shared.budget.lock().expect("poisoned");
547 budget.refill(now);
548 let effective = budget.effective_budget_per_sec;
549 let reason = if !budget.config.enabled {
550 EffectiveReason::Disabled
551 } else if budget.config.budget_per_sec_cap.is_some()
552 && budget
553 .config
554 .budget_per_sec_cap
555 .is_some_and(|c| (effective - c).abs() < 1e-9)
556 {
557 EffectiveReason::ConfigCap
558 } else if budget.ongoing_capacity.is_some() {
559 EffectiveReason::Ongoing
560 } else if budget.startup_capacity.is_some() {
561 EffectiveReason::StartupProbe
562 } else {
563 EffectiveReason::DefaultCapacity
564 };
565 let (queued, last_enq) = {
566 let q = self.shared.queue.lock().expect("poisoned");
567 (q.bytes, q.last_enqueue)
568 };
569 let early = budget.allow_early_flush(now, queued, last_enq);
570 let max_linger = self.shared.flush_config.linger;
571 let eff_linger = if !budget.config.enabled {
572 max_linger
573 } else if early || max_linger.is_zero() {
574 Duration::ZERO
575 } else {
576 max_linger
577 };
578 PipelineSnapshot {
579 budget_per_sec: EffectiveKnob {
580 configured: budget.config.budget_per_sec_cap,
581 startup_measured: budget.startup_capacity,
582 ongoing_measured: budget.ongoing_capacity,
583 effective,
584 reason,
585 },
586 effective_linger_ms: EffectiveKnob {
587 configured: Some(max_linger.as_millis() as u64),
588 startup_measured: None,
589 ongoing_measured: None,
590 effective: eff_linger.as_millis() as u64,
591 reason: if early {
592 EffectiveReason::Ongoing
593 } else {
594 EffectiveReason::Configured
595 },
596 },
597 max_linger_ms: max_linger.as_millis() as u64,
598 token_fill_ratio: budget.fill_ratio(),
599 media_ops_total: budget.media_ops_total,
600 overdraft_total: budget.overdraft_total,
601 flushes_total: budget.flushes_total,
602 undersized_deadline_flushes: budget.undersized_deadline_flushes,
603 predicted_media_ops: budget.predicted_media_ops,
604 budget_enabled: budget.config.enabled,
605 budget_mode: budget.config.mode,
606 }
607 }
608}
609
610pub async fn reap_orphans(
616 blob: &dyn BlobStore,
617 data_prefix: &str,
618 live: &HashSet<String>,
619) -> Result<Vec<String>, ObjectLogError> {
620 let keys = blob.list(data_prefix).await?;
621 let mut deleted = Vec::new();
622 for key in keys {
623 if live.contains(&key) {
624 continue;
625 }
626 blob.delete(&key).await?;
627 deleted.push(key);
628 }
629 Ok(deleted)
630}
631
632impl<S: Sequencer> Drop for LogEngine<S> {
633 fn drop(&mut self) {
634 {
635 let mut q = self.shared.queue.lock().expect("poisoned");
636 q.shutdown = true;
637 }
638 self.shared.cv.notify_all();
639 if let Some(t) = self.flush_thread.take() {
640 let _ = t.join();
641 }
642 }
643}
644
645fn flush_loop<S>(
646 shared: Arc<Shared<S::Meta>>,
647 blob: Arc<dyn BlobStore>,
648 sequencer: Arc<S>,
649 config: FlushConfig,
650 prefix: String,
651) where
652 S: Sequencer + 'static,
653 S::Meta: Send + 'static,
654{
655 let max_inflight = config.max_inflight_flushes.max(1);
656 let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
657 .ok()
658 .and_then(|v| v.parse::<usize>().ok())
659 .filter(|v| *v > 0)
660 .unwrap_or_else(|| max_inflight.min(8));
661 let rt = tokio::runtime::Builder::new_multi_thread()
662 .worker_threads(worker_threads)
663 .enable_all()
664 .build()
665 .expect("flush runtime");
666 let mut counter: u64 = 0;
667 let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
668 let mut active_puts = 0usize;
669 let mut shutdown = false;
670
671 loop {
672 while !shutdown && active_puts < max_inflight {
673 let wait_for_more = pending.is_empty();
674 match take_batch(&shared, config, wait_for_more) {
675 TakeBatch::Batch(batch) => {
676 counter += 1;
677 let concurrent = max_inflight > 1;
678 let work = start_flush_work(&rt, &blob, &prefix, counter, batch, concurrent);
679 if concurrent {
680 active_puts += 1;
681 }
682 pending.push_back(work);
683 }
684 TakeBatch::Empty => break,
685 TakeBatch::Shutdown => shutdown = true,
686 }
687 }
688
689 let mut made_progress = false;
690 for work in pending.iter_mut() {
691 let Some(put) = work.put.as_ref() else {
692 continue;
693 };
694 if !put.is_finished() {
695 continue;
696 }
697 let put = work.put.take().expect("put handle exists");
698 let elapsed = work.put_started.elapsed();
699 work.put_result = Some(match rt.block_on(put) {
700 Ok(Ok(())) => Ok(elapsed),
701 Ok(Err(e)) => Err(e),
702 Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
703 "flush task failed: {e}"
704 ))),
705 });
706 active_puts = active_puts.saturating_sub(1);
707 made_progress = true;
708 }
709
710 while pending
711 .front()
712 .is_some_and(|work| work.put_result.is_some())
713 {
714 let work = pending.pop_front().expect("front exists");
715 let released = finish_flush_work(&shared, &blob, &sequencer, work);
716 let mut q = shared.queue.lock().expect("poisoned");
717 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
718 shared.cv.notify_all();
719 made_progress = true;
720 }
721
722 if pending.is_empty() {
723 {
724 let mut q = shared.queue.lock().expect("poisoned");
725 if q.items.is_empty() && q.bytes_in_use == 0 {
726 notify_flush_waiters(&mut q);
729 }
730 }
731 shared.cv.notify_all();
732 if shutdown {
733 let mut q = shared.queue.lock().expect("poisoned");
735 for (_, tx) in q.flush_waiters.drain(..) {
736 let _ = tx.send(Err(ObjectLogError::Sequencer(
737 "engine shutting down".into(),
738 )));
739 }
740 return;
741 }
742 continue;
743 }
744
745 if !made_progress {
746 std::thread::sleep(Duration::from_millis(1));
747 }
748 }
749}
750
751fn effective_linger<M>(
755 shared: &Shared<M>,
756 config: FlushConfig,
757 queued_bytes: usize,
758 last_enqueue: Option<Instant>,
759 oldest_enqueue: Option<Instant>,
760) -> Duration {
761 if config.linger.is_zero() {
762 return Duration::ZERO;
763 }
764 let now = Instant::now();
765 if config.budget.enabled {
766 let mut budget = shared.budget.lock().expect("poisoned");
767 budget.refill(now);
768 if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
769 budget.note_early_flush(now);
770 return Duration::ZERO;
771 }
772 }
773 if let Some(oldest) = oldest_enqueue {
775 let deadline = oldest + config.linger;
776 if now >= deadline {
777 return Duration::ZERO;
778 }
779 let until_deadline = deadline.saturating_duration_since(now);
780 if config.budget.enabled
782 && !config.budget.early_flush_idle.is_zero()
783 && let Some(last) = last_enqueue
784 {
785 let idle_at = last + config.budget.early_flush_idle;
786 if idle_at > now && idle_at < deadline {
787 return idle_at.saturating_duration_since(now);
788 }
789 }
790 return until_deadline;
791 }
792 config.linger
793}
794
795fn take_batch<M>(
796 shared: &Arc<Shared<M>>,
797 config: FlushConfig,
798 wait_when_empty: bool,
799) -> TakeBatch<M> {
800 let mut q = shared.queue.lock().expect("poisoned");
801 loop {
802 if q.items.is_empty() {
803 if q.shutdown {
804 return TakeBatch::Shutdown;
805 }
806 if !wait_when_empty {
807 return TakeBatch::Empty;
808 }
809 let linger = {
810 let queued = q.bytes;
811 let last_enq = q.last_enqueue;
812 let oldest = q.oldest_enqueue;
813 drop(q);
814 let l = effective_linger(shared, config, queued, last_enq, oldest);
815 q = shared.queue.lock().expect("poisoned");
816 l
817 };
818 if linger.is_zero() {
819 q = shared.cv.wait(q).expect("poisoned");
820 } else {
821 let (guard, timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
822 q = guard;
823 if timeout.timed_out() && q.items.is_empty() {
824 return TakeBatch::Empty;
825 }
826 }
827 continue;
828 }
829
830 let linger = {
831 let queued = q.bytes;
832 let last_enq = q.last_enqueue;
833 let oldest = q.oldest_enqueue;
834 drop(q);
835 let l = effective_linger(shared, config, queued, last_enq, oldest);
836 q = shared.queue.lock().expect("poisoned");
837 l
838 };
839
840 let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
841 let force = q.force_flush;
842 let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
843 if triggered {
844 if !size_trigger && !q.shutdown && !q.items.is_empty() {
845 let mut budget = shared.budget.lock().expect("poisoned");
846 if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
847 budget.undersized_deadline_flushes += 1;
848 }
849 }
850 break;
851 }
852
853 let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
856 q = guard;
857 }
858
859 let force_drain = q.force_flush;
860 let mut items = Vec::new();
861 let mut bytes = 0usize;
862 while let Some(item) = q.items.pop_front() {
865 q.bytes = q.bytes.saturating_sub(item.payload.len());
866 bytes += item.payload.len();
867 items.push(item);
868 if bytes >= config.max_bytes {
869 break;
870 }
871 if !force_drain && items.len() >= config.max_batches {
872 break;
873 }
874 }
875 q.oldest_enqueue = if q.items.is_empty() {
876 None
877 } else {
878 Some(Instant::now())
880 };
881 if force_drain {
883 q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
884 }
885 if items.is_empty() {
886 TakeBatch::Empty
887 } else {
888 TakeBatch::Batch(items)
889 }
890}
891
892fn notify_flush_waiters<M>(q: &mut Queue<M>) {
893 let done = q.completed_through;
894 let mut i = 0;
895 while i < q.flush_waiters.len() {
896 if q.flush_waiters[i].0 <= done {
897 let (_, tx) = q.flush_waiters.swap_remove(i);
898 let _ = tx.send(Ok(()));
899 } else {
900 i += 1;
901 }
902 }
903 if q.flush_waiters.is_empty() {
904 q.force_flush = false;
905 } else if q.items.is_empty() && q.bytes_in_use == 0 {
906 q.force_flush = true;
908 } else {
909 q.force_flush = true;
910 }
911}
912
913fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
914 for (_, tx) in responders.iter_mut() {
915 if let Some(tx) = tx.take() {
916 let _ = tx.send(Err(err.clone()));
917 }
918 }
919}
920
921fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
922 for (durability, tx) in responders.iter_mut() {
923 if *durability == Durability::Durable
924 && let Some(tx) = tx.take()
925 {
926 let _ = tx.send(Ok(AppendOutcome {
927 base_offset: None,
928 last_offset: None,
929 durable: true,
930 sequenced: false,
931 }));
932 }
933 }
934}
935
936fn prepare_flush_work<M>(
937 prefix: &str,
938 counter: u64,
939 mut batch: Vec<Pending<M>>,
940) -> (FlushWork<M>, String, Vec<Bytes>) {
941 let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
942 let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
943 let key = format!("{prefix}{counter:020}");
944 let mut offset = 0usize;
945 for p in &batch {
946 let start = offset as u32;
947 offset += p.payload.len();
948 chunks.push(p.payload.clone());
949 locations.push(BatchLocation {
950 object_id: key.clone(),
951 byte_start: start,
952 byte_len: p.payload.len() as u32,
953 });
954 }
955 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
956 eprintln!(
957 "object-log seal: key={key} batches={} bytes={offset}",
958 batch.len()
959 );
960 }
961 let responders: Vec<(Durability, Option<Responder>)> = batch
962 .iter_mut()
963 .map(|p| (p.durability, p.responder.take()))
964 .collect();
965 let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
966 let work = FlushWork {
967 batch,
968 locations,
969 responders,
970 bytes: offset,
971 max_seq,
972 put: None,
973 put_started: Instant::now(),
974 put_result: None,
975 };
976 (work, key, chunks)
977}
978
979fn start_flush_work<M>(
985 rt: &tokio::runtime::Runtime,
986 blob: &Arc<dyn BlobStore>,
987 prefix: &str,
988 counter: u64,
989 batch: Vec<Pending<M>>,
990 concurrent: bool,
991) -> FlushWork<M> {
992 let (mut work, key, chunks) = prepare_flush_work(prefix, counter, batch);
993 let blob = Arc::clone(blob);
994 let _ = blob.take_media_op_stats();
995 work.put_started = Instant::now();
996 if concurrent {
997 work.put =
998 Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
999 } else {
1000 let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1001 let elapsed = work.put_started.elapsed();
1002 work.put_result = Some(match put_result {
1003 Ok(()) => Ok(elapsed),
1004 Err(e) => Err(e),
1005 });
1006 }
1007 work
1008}
1009
1010fn finish_flush_work<S, M>(
1011 shared: &Arc<Shared<M>>,
1012 blob: &Arc<dyn BlobStore>,
1013 sequencer: &Arc<S>,
1014 mut work: FlushWork<S::Meta>,
1015) -> usize
1016where
1017 S: Sequencer + 'static,
1018 S::Meta: Send + 'static,
1019{
1020 let release_bytes = work.bytes;
1021 let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1024 let put_elapsed = match work.put_result.take().expect("put result is ready") {
1025 Ok(elapsed) => elapsed,
1026 Err(e) => {
1027 send_storage_error(&mut work.responders, e.clone());
1028 let mut q = shared.queue.lock().expect("poisoned");
1029 let done = work.max_seq;
1031 if done >= q.completed_through {
1032 q.completed_through = done;
1033 }
1034 let mut i = 0;
1035 while i < q.flush_waiters.len() {
1036 if q.flush_waiters[i].0 <= done {
1037 let (_, tx) = q.flush_waiters.swap_remove(i);
1038 let _ = tx.send(Err(e.clone()));
1039 } else {
1040 i += 1;
1041 }
1042 }
1043 shared.cv.notify_all();
1044 return release_bytes;
1045 }
1046 };
1047
1048 let mut media_ops = blob.take_media_op_stats().map(|s| s.media_ops).unwrap_or(1); send_durable_acks(&mut work.responders);
1053
1054 let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1056 .batch
1057 .iter()
1058 .zip(work.locations.iter())
1059 .map(|(p, loc)| CommitBatch {
1060 partition: p.partition.clone(),
1061 record_count: p.record_count,
1062 location: loc.clone(),
1063 meta: &p.meta,
1064 })
1065 .collect();
1066
1067 let commit_started = timing.then(Instant::now);
1068 let _ = blob.take_media_op_stats();
1071 match sequencer.commit(&commit_batches) {
1072 Ok(outcomes) => {
1073 if let Some(commit_started) = commit_started {
1074 eprintln!(
1075 "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={}",
1076 work.bytes,
1077 commit_batches.len(),
1078 put_elapsed.as_millis(),
1079 commit_started.elapsed().as_millis()
1080 );
1081 }
1082 if let Some(s) = blob.take_media_op_stats() {
1083 media_ops = media_ops.saturating_add(s.media_ops);
1084 }
1085 for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1086 if let Some(tx) = tx.take() {
1087 let (base, last) = match outcome {
1088 CommitOutcome::Assigned {
1089 base_offset,
1090 record_count,
1091 } => (
1092 Some(base_offset),
1093 Some(base_offset + record_count as i64 - 1),
1094 ),
1095 CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1096 };
1097 let _ = tx.send(Ok(AppendOutcome {
1098 base_offset: base,
1099 last_offset: last,
1100 durable: true,
1101 sequenced: true,
1102 }));
1103 }
1104 }
1105 }
1106 Err(e) => {
1107 if let Some(s) = blob.take_media_op_stats() {
1108 media_ops = media_ops.saturating_add(s.media_ops);
1109 }
1110 send_storage_error(&mut work.responders, e);
1111 }
1112 }
1113
1114 {
1115 let mut budget = shared.budget.lock().expect("poisoned");
1116 budget.consume_after_flush(media_ops, Instant::now());
1117 }
1118 {
1119 let mut q = shared.queue.lock().expect("poisoned");
1120 if work.max_seq >= q.completed_through {
1121 q.completed_through = work.max_seq;
1122 }
1123 notify_flush_waiters(&mut q);
1124 }
1125 shared.cv.notify_all();
1126
1127 release_bytes
1128}