1use crate::budget::{
4 BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{BlobStore, CommitBatch, CommitOutcome, ObjectLogError, PartitionKey, Sequencer};
8use bytes::Bytes;
9use std::collections::VecDeque;
10use std::sync::{Arc, Condvar, Mutex};
11use std::thread::JoinHandle;
12use std::time::{Duration, Instant};
13use tokio::sync::oneshot;
14use tokio::task::JoinHandle as TokioJoinHandle;
15
16const STORAGE_RETRY_ATTEMPTS: usize = 5;
17const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
18
19#[derive(Clone, Copy, Debug, PartialEq, Eq)]
21pub enum Durability {
22 Buffered,
25 Durable,
28 Sequenced,
31}
32
33#[derive(Clone, Copy, Debug)]
35pub struct FlushConfig {
36 pub max_bytes: usize,
41 pub max_batches: usize,
43 pub linger: Duration,
49 pub max_inflight_flushes: usize,
52 pub max_buffered_bytes: usize,
55 pub budget: BudgetConfig,
57}
58
59impl Default for FlushConfig {
60 fn default() -> Self {
61 Self {
62 max_bytes: 1024 * 1024 * 1024,
64 max_batches: 10_000,
65 linger: Duration::from_millis(50),
66 max_inflight_flushes: 1,
68 max_buffered_bytes: 2 * 1024 * 1024 * 1024,
69 budget: BudgetConfig::default(),
70 }
71 }
72}
73
74#[derive(Clone, Debug, PartialEq, Eq)]
76pub struct AppendOutcome {
77 pub base_offset: Option<i64>,
79 pub last_offset: Option<i64>,
81 pub durable: bool,
83 pub sequenced: bool,
85}
86
87#[derive(Clone, Debug, PartialEq, Eq)]
89pub struct FetchedBatch {
90 pub base_offset: i64,
92 pub record_count: i32,
94 pub payload: Bytes,
96}
97
98#[derive(Clone, Debug, PartialEq, Eq)]
100pub struct BufferStats {
101 pub queued_bytes: usize,
103 pub bytes_in_use: usize,
105 pub queued_batches: usize,
107 pub max_buffered_bytes: usize,
109}
110
111type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
112
113fn retryable_storage_error(err: &ObjectLogError) -> bool {
114 matches!(err, ObjectLogError::StorageUnavailable(_))
115}
116
117async fn retry_delay(attempt: usize) {
118 let multiplier = 1u32 << attempt.min(5);
119 tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
120}
121
122async fn put_chunks_with_retries(
123 blob: &Arc<dyn BlobStore>,
124 key: &str,
125 chunks: Vec<Bytes>,
126) -> Result<(), ObjectLogError> {
127 let mut attempt = 0usize;
129 loop {
130 match blob.put_chunks(key, chunks.clone()).await {
131 Ok(()) => return Ok(()),
132 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
133 retry_delay(attempt).await;
134 attempt += 1;
135 }
136 Err(err) => return Err(err),
137 }
138 }
139}
140
141async fn get_range_with_retries(
142 blob: &Arc<dyn BlobStore>,
143 key: &str,
144 range: std::ops::Range<u64>,
145) -> Result<Option<Bytes>, ObjectLogError> {
146 let mut attempt = 0usize;
147 loop {
148 match blob.get_range(key, range.clone()).await {
149 Ok(bytes) => return Ok(bytes),
150 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
151 retry_delay(attempt).await;
152 attempt += 1;
153 }
154 Err(err) => return Err(err),
155 }
156 }
157}
158
159struct Pending<M> {
160 partition: PartitionKey,
161 record_count: i32,
162 payload: Bytes,
163 meta: M,
164 durability: Durability,
165 responder: Option<Responder>,
166 seq: u64,
168}
169
170struct FlushWork<M> {
171 batch: Vec<Pending<M>>,
172 locations: Vec<BatchLocation>,
173 responders: Vec<(Durability, Option<Responder>)>,
174 bytes: usize,
175 max_seq: u64,
177 put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
178 put_started: Instant,
179 put_result: Option<Result<Duration, ObjectLogError>>,
180}
181
182enum TakeBatch<M> {
183 Batch(Vec<Pending<M>>),
184 Empty,
185 Shutdown,
186}
187
188type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
189
190struct Queue<M> {
191 items: VecDeque<Pending<M>>,
192 bytes: usize,
193 bytes_in_use: usize,
194 shutdown: bool,
195 next_seq: u64,
197 completed_through: u64,
199 force_flush: bool,
201 flush_waiters: Vec<FlushWaiter>,
203 last_enqueue: Option<Instant>,
205 oldest_enqueue: Option<Instant>,
207}
208
209struct Shared<M> {
210 queue: Mutex<Queue<M>>,
211 cv: Condvar,
212 max_buffered_bytes: usize,
213 budget: Mutex<BudgetRuntime>,
215 flush_config: FlushConfig,
217}
218
219pub struct LogEngine<S: Sequencer> {
228 shared: Arc<Shared<S::Meta>>,
229 blob: Arc<dyn BlobStore>,
230 sequencer: Arc<S>,
231 flush_thread: Option<JoinHandle<()>>,
232}
233
234impl<S> LogEngine<S>
235where
236 S: Sequencer + 'static,
237 S::Meta: Send + 'static,
238{
239 pub fn new(
243 blob: Arc<dyn BlobStore>,
244 sequencer: Arc<S>,
245 config: FlushConfig,
246 key_prefix: impl Into<String>,
247 ) -> Self {
248 if let Err(msg) = config.budget.validate() {
249 panic!("invalid FlushConfig.budget: {msg}");
250 }
251 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
252 eprintln!("object-log flush config: {config:?}");
253 }
254 let budget_rt = BudgetRuntime::new(config.budget);
255 let shared = Arc::new(Shared {
256 queue: Mutex::new(Queue {
257 items: VecDeque::new(),
258 bytes: 0,
259 bytes_in_use: 0,
260 shutdown: false,
261 next_seq: 1, completed_through: 0,
263 force_flush: false,
264 flush_waiters: Vec::new(),
265 last_enqueue: None,
266 oldest_enqueue: None,
267 }),
268 cv: Condvar::new(),
269 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
270 budget: Mutex::new(budget_rt),
271 flush_config: config,
272 });
273 let flush_thread = {
274 let shared = Arc::clone(&shared);
275 let blob = Arc::clone(&blob);
276 let sequencer = Arc::clone(&sequencer);
277 let prefix = key_prefix.into();
278 std::thread::Builder::new()
279 .name("object-log-flush".into())
280 .spawn(move || flush_loop(shared, blob, sequencer, config, prefix))
281 .expect("spawn flush thread")
282 };
283 Self {
284 shared,
285 blob,
286 sequencer,
287 flush_thread: Some(flush_thread),
288 }
289 }
290
291 pub async fn produce(
293 &self,
294 partition: PartitionKey,
295 payload: Bytes,
296 record_count: i32,
297 meta: S::Meta,
298 durability: Durability,
299 ) -> Result<AppendOutcome, ObjectLogError> {
300 if record_count <= 0 {
301 return Err(ObjectLogError::InvalidBatch(
302 "record_count must be > 0".into(),
303 ));
304 }
305 if matches!(durability, Durability::Buffered) {
306 self.enqueue(Pending {
307 partition,
308 record_count,
309 payload,
310 meta,
311 durability,
312 responder: None,
313 seq: 0, })?;
315 return Ok(AppendOutcome {
316 base_offset: None,
317 last_offset: None,
318 durable: false,
319 sequenced: false,
320 });
321 }
322 {
324 let mut budget = self.shared.budget.lock().expect("poisoned");
325 if budget.config.enabled
326 && budget.config.mode == BudgetMode::FailClosed
327 && !budget.reserve_for_fail_closed(Instant::now())
328 {
329 return Err(ObjectLogError::BudgetExceeded(
330 "insufficient durable-ops budget to admit produce".into(),
331 ));
332 }
333 }
334 if self.shared.flush_config.budget.enabled
336 && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
337 {
338 let timeout = self.shared.flush_config.budget.admission_timeout;
339 let deadline = Instant::now() + timeout;
340 loop {
341 let mut budget = self.shared.budget.lock().expect("poisoned");
342 if budget.can_admit_now(Instant::now()) {
343 break;
344 }
345 drop(budget);
346 if Instant::now() >= deadline {
347 return Err(ObjectLogError::BudgetExceeded(
348 "timed out waiting for durable-ops budget".into(),
349 ));
350 }
351 std::thread::sleep(Duration::from_millis(1));
352 }
353 }
354 let (tx, rx) = oneshot::channel();
355 self.enqueue(Pending {
356 partition,
357 record_count,
358 payload,
359 meta,
360 durability,
361 responder: Some(tx),
362 seq: 0,
363 })?;
364 rx.await
365 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
366 }
367
368 pub async fn flush(&self) -> Result<(), ObjectLogError> {
374 let rx = {
375 let mut q = self.shared.queue.lock().expect("poisoned");
376 if q.shutdown {
377 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
378 }
379 if q.next_seq == 1 {
380 return Ok(());
382 }
383 let barrier = q.next_seq - 1;
384 if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
385 return Ok(());
386 }
387 let (tx, rx) = oneshot::channel();
388 q.force_flush = true;
389 q.flush_waiters.push((barrier, tx));
390 self.shared.cv.notify_all();
391 rx
392 };
393 rx.await
394 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
395 }
396
397 fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
398 let item_len = item.payload.len();
399 let max_buffered_bytes = self.shared.max_buffered_bytes;
400 let shared = Arc::clone(&self.shared);
401 let mut q = shared.queue.lock().expect("poisoned");
402 while !q.shutdown
403 && q.bytes_in_use > 0
404 && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
405 {
406 q = shared.cv.wait(q).expect("poisoned");
407 }
408 if q.shutdown {
409 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
410 }
411 let now = Instant::now();
412 item.seq = q.next_seq;
413 q.next_seq = q.next_seq.saturating_add(1);
414 q.bytes += item_len;
415 q.bytes_in_use += item_len;
416 q.last_enqueue = Some(now);
417 if q.oldest_enqueue.is_none() {
418 q.oldest_enqueue = Some(now);
419 }
420 q.items.push_back(item);
421 shared.cv.notify_all();
422 Ok(())
423 }
424
425 pub async fn fetch(
427 &self,
428 partition: &PartitionKey,
429 offset: i64,
430 max_bytes: usize,
431 ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
432 let entries = self.sequencer.lookup(partition, offset)?;
433 let mut out = Vec::new();
434 let mut total = 0usize;
435 for e in entries {
436 if total >= max_bytes && !out.is_empty() {
437 break;
438 }
439 let start = e.location.byte_start as u64;
440 let end = start + e.location.byte_len as u64;
441 let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
442 .await?
443 .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
444 total += bytes.len();
445 out.push(FetchedBatch {
446 base_offset: e.base_offset,
447 record_count: e.record_count,
448 payload: bytes,
449 });
450 }
451 Ok(out)
452 }
453
454 pub async fn truncate_before(
457 &self,
458 partition: &PartitionKey,
459 offset: i64,
460 ) -> Result<(), ObjectLogError> {
461 let dead = self.sequencer.truncate_before(partition, offset)?;
462 for object_id in dead {
463 self.blob.delete(&object_id).await?;
464 }
465 Ok(())
466 }
467
468 pub fn buffer_stats(&self) -> BufferStats {
470 let q = self.shared.queue.lock().expect("poisoned");
471 BufferStats {
472 queued_bytes: q.bytes,
473 bytes_in_use: q.bytes_in_use,
474 queued_batches: q.items.len(),
475 max_buffered_bytes: self.shared.max_buffered_bytes,
476 }
477 }
478
479 pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
481 let now = Instant::now();
482 let mut budget = self.shared.budget.lock().expect("poisoned");
483 budget.refill(now);
484 let effective = budget.effective_budget_per_sec;
485 let reason = if !budget.config.enabled {
486 EffectiveReason::Disabled
487 } else if budget.config.budget_per_sec_cap.is_some()
488 && budget
489 .config
490 .budget_per_sec_cap
491 .is_some_and(|c| (effective - c).abs() < 1e-9)
492 {
493 EffectiveReason::ConfigCap
494 } else if budget.ongoing_capacity.is_some() {
495 EffectiveReason::Ongoing
496 } else if budget.startup_capacity.is_some() {
497 EffectiveReason::StartupProbe
498 } else {
499 EffectiveReason::DefaultCapacity
500 };
501 let (queued, last_enq) = {
502 let q = self.shared.queue.lock().expect("poisoned");
503 (q.bytes, q.last_enqueue)
504 };
505 let early = budget.allow_early_flush(now, queued, last_enq);
506 let max_linger = self.shared.flush_config.linger;
507 let eff_linger = if !budget.config.enabled {
508 max_linger
509 } else if early || max_linger.is_zero() {
510 Duration::ZERO
511 } else {
512 max_linger
513 };
514 PipelineSnapshot {
515 budget_per_sec: EffectiveKnob {
516 configured: budget.config.budget_per_sec_cap,
517 startup_measured: budget.startup_capacity,
518 ongoing_measured: budget.ongoing_capacity,
519 effective,
520 reason,
521 },
522 effective_linger_ms: EffectiveKnob {
523 configured: Some(max_linger.as_millis() as u64),
524 startup_measured: None,
525 ongoing_measured: None,
526 effective: eff_linger.as_millis() as u64,
527 reason: if early {
528 EffectiveReason::Ongoing
529 } else {
530 EffectiveReason::Configured
531 },
532 },
533 max_linger_ms: max_linger.as_millis() as u64,
534 token_fill_ratio: budget.fill_ratio(),
535 media_ops_total: budget.media_ops_total,
536 overdraft_total: budget.overdraft_total,
537 flushes_total: budget.flushes_total,
538 undersized_deadline_flushes: budget.undersized_deadline_flushes,
539 predicted_media_ops: budget.predicted_media_ops,
540 budget_enabled: budget.config.enabled,
541 budget_mode: budget.config.mode,
542 }
543 }
544}
545
546impl<S: Sequencer> Drop for LogEngine<S> {
547 fn drop(&mut self) {
548 {
549 let mut q = self.shared.queue.lock().expect("poisoned");
550 q.shutdown = true;
551 }
552 self.shared.cv.notify_all();
553 if let Some(t) = self.flush_thread.take() {
554 let _ = t.join();
555 }
556 }
557}
558
559fn flush_loop<S>(
560 shared: Arc<Shared<S::Meta>>,
561 blob: Arc<dyn BlobStore>,
562 sequencer: Arc<S>,
563 config: FlushConfig,
564 prefix: String,
565) where
566 S: Sequencer + 'static,
567 S::Meta: Send + 'static,
568{
569 let max_inflight = config.max_inflight_flushes.max(1);
570 let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
571 .ok()
572 .and_then(|v| v.parse::<usize>().ok())
573 .filter(|v| *v > 0)
574 .unwrap_or_else(|| max_inflight.min(8));
575 let rt = tokio::runtime::Builder::new_multi_thread()
576 .worker_threads(worker_threads)
577 .enable_all()
578 .build()
579 .expect("flush runtime");
580 let mut counter: u64 = 0;
581 let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
582 let mut active_puts = 0usize;
583 let mut shutdown = false;
584
585 loop {
586 while !shutdown && active_puts < max_inflight {
587 let wait_for_more = pending.is_empty();
588 match take_batch(&shared, config, wait_for_more) {
589 TakeBatch::Batch(batch) => {
590 counter += 1;
591 let concurrent = max_inflight > 1;
592 let work = start_flush_work(&rt, &blob, &prefix, counter, batch, concurrent);
593 if concurrent {
594 active_puts += 1;
595 }
596 pending.push_back(work);
597 }
598 TakeBatch::Empty => break,
599 TakeBatch::Shutdown => shutdown = true,
600 }
601 }
602
603 let mut made_progress = false;
604 for work in pending.iter_mut() {
605 let Some(put) = work.put.as_ref() else {
606 continue;
607 };
608 if !put.is_finished() {
609 continue;
610 }
611 let put = work.put.take().expect("put handle exists");
612 let elapsed = work.put_started.elapsed();
613 work.put_result = Some(match rt.block_on(put) {
614 Ok(Ok(())) => Ok(elapsed),
615 Ok(Err(e)) => Err(e),
616 Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
617 "flush task failed: {e}"
618 ))),
619 });
620 active_puts = active_puts.saturating_sub(1);
621 made_progress = true;
622 }
623
624 while pending
625 .front()
626 .is_some_and(|work| work.put_result.is_some())
627 {
628 let work = pending.pop_front().expect("front exists");
629 let released = finish_flush_work(&shared, &blob, &sequencer, work);
630 let mut q = shared.queue.lock().expect("poisoned");
631 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
632 shared.cv.notify_all();
633 made_progress = true;
634 }
635
636 if pending.is_empty() {
637 {
638 let mut q = shared.queue.lock().expect("poisoned");
639 if q.items.is_empty() && q.bytes_in_use == 0 {
640 notify_flush_waiters(&mut q);
643 }
644 }
645 shared.cv.notify_all();
646 if shutdown {
647 let mut q = shared.queue.lock().expect("poisoned");
649 for (_, tx) in q.flush_waiters.drain(..) {
650 let _ = tx.send(Err(ObjectLogError::Sequencer(
651 "engine shutting down".into(),
652 )));
653 }
654 return;
655 }
656 continue;
657 }
658
659 if !made_progress {
660 std::thread::sleep(Duration::from_millis(1));
661 }
662 }
663}
664
665fn effective_linger<M>(
669 shared: &Shared<M>,
670 config: FlushConfig,
671 queued_bytes: usize,
672 last_enqueue: Option<Instant>,
673 oldest_enqueue: Option<Instant>,
674) -> Duration {
675 if config.linger.is_zero() {
676 return Duration::ZERO;
677 }
678 let now = Instant::now();
679 if config.budget.enabled {
680 let mut budget = shared.budget.lock().expect("poisoned");
681 budget.refill(now);
682 if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
683 budget.note_early_flush(now);
684 return Duration::ZERO;
685 }
686 }
687 if let Some(oldest) = oldest_enqueue {
689 let deadline = oldest + config.linger;
690 if now >= deadline {
691 return Duration::ZERO;
692 }
693 let until_deadline = deadline.saturating_duration_since(now);
694 if config.budget.enabled
696 && !config.budget.early_flush_idle.is_zero()
697 && let Some(last) = last_enqueue
698 {
699 let idle_at = last + config.budget.early_flush_idle;
700 if idle_at > now && idle_at < deadline {
701 return idle_at.saturating_duration_since(now);
702 }
703 }
704 return until_deadline;
705 }
706 config.linger
707}
708
709fn take_batch<M>(
710 shared: &Arc<Shared<M>>,
711 config: FlushConfig,
712 wait_when_empty: bool,
713) -> TakeBatch<M> {
714 let mut q = shared.queue.lock().expect("poisoned");
715 loop {
716 if q.items.is_empty() {
717 if q.shutdown {
718 return TakeBatch::Shutdown;
719 }
720 if !wait_when_empty {
721 return TakeBatch::Empty;
722 }
723 let linger = {
724 let queued = q.bytes;
725 let last_enq = q.last_enqueue;
726 let oldest = q.oldest_enqueue;
727 drop(q);
728 let l = effective_linger(shared, config, queued, last_enq, oldest);
729 q = shared.queue.lock().expect("poisoned");
730 l
731 };
732 if linger.is_zero() {
733 q = shared.cv.wait(q).expect("poisoned");
734 } else {
735 let (guard, timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
736 q = guard;
737 if timeout.timed_out() && q.items.is_empty() {
738 return TakeBatch::Empty;
739 }
740 }
741 continue;
742 }
743
744 let linger = {
745 let queued = q.bytes;
746 let last_enq = q.last_enqueue;
747 let oldest = q.oldest_enqueue;
748 drop(q);
749 let l = effective_linger(shared, config, queued, last_enq, oldest);
750 q = shared.queue.lock().expect("poisoned");
751 l
752 };
753
754 let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
755 let force = q.force_flush;
756 let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
757 if triggered {
758 if !size_trigger && !q.shutdown && !q.items.is_empty() {
759 let mut budget = shared.budget.lock().expect("poisoned");
760 if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
761 budget.undersized_deadline_flushes += 1;
762 }
763 }
764 break;
765 }
766
767 let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
770 q = guard;
771 }
772
773 let force_drain = q.force_flush;
774 let mut items = Vec::new();
775 let mut bytes = 0usize;
776 while let Some(item) = q.items.pop_front() {
779 q.bytes = q.bytes.saturating_sub(item.payload.len());
780 bytes += item.payload.len();
781 items.push(item);
782 if bytes >= config.max_bytes {
783 break;
784 }
785 if !force_drain && items.len() >= config.max_batches {
786 break;
787 }
788 }
789 q.oldest_enqueue = if q.items.is_empty() {
790 None
791 } else {
792 Some(Instant::now())
794 };
795 if force_drain {
797 q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
798 }
799 if items.is_empty() {
800 TakeBatch::Empty
801 } else {
802 TakeBatch::Batch(items)
803 }
804}
805
806fn notify_flush_waiters<M>(q: &mut Queue<M>) {
807 let done = q.completed_through;
808 let mut i = 0;
809 while i < q.flush_waiters.len() {
810 if q.flush_waiters[i].0 <= done {
811 let (_, tx) = q.flush_waiters.swap_remove(i);
812 let _ = tx.send(Ok(()));
813 } else {
814 i += 1;
815 }
816 }
817 if q.flush_waiters.is_empty() {
818 q.force_flush = false;
819 } else if q.items.is_empty() && q.bytes_in_use == 0 {
820 q.force_flush = true;
822 } else {
823 q.force_flush = true;
824 }
825}
826
827fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
828 for (_, tx) in responders.iter_mut() {
829 if let Some(tx) = tx.take() {
830 let _ = tx.send(Err(err.clone()));
831 }
832 }
833}
834
835fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
836 for (durability, tx) in responders.iter_mut() {
837 if *durability == Durability::Durable
838 && let Some(tx) = tx.take()
839 {
840 let _ = tx.send(Ok(AppendOutcome {
841 base_offset: None,
842 last_offset: None,
843 durable: true,
844 sequenced: false,
845 }));
846 }
847 }
848}
849
850fn prepare_flush_work<M>(
851 prefix: &str,
852 counter: u64,
853 mut batch: Vec<Pending<M>>,
854) -> (FlushWork<M>, String, Vec<Bytes>) {
855 let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
856 let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
857 let key = format!("{prefix}{counter:020}");
858 let mut offset = 0usize;
859 for p in &batch {
860 let start = offset as u32;
861 offset += p.payload.len();
862 chunks.push(p.payload.clone());
863 locations.push(BatchLocation {
864 object_id: key.clone(),
865 byte_start: start,
866 byte_len: p.payload.len() as u32,
867 });
868 }
869 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
870 eprintln!(
871 "object-log seal: key={key} batches={} bytes={offset}",
872 batch.len()
873 );
874 }
875 let responders: Vec<(Durability, Option<Responder>)> = batch
876 .iter_mut()
877 .map(|p| (p.durability, p.responder.take()))
878 .collect();
879 let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
880 let work = FlushWork {
881 batch,
882 locations,
883 responders,
884 bytes: offset,
885 max_seq,
886 put: None,
887 put_started: Instant::now(),
888 put_result: None,
889 };
890 (work, key, chunks)
891}
892
893fn start_flush_work<M>(
899 rt: &tokio::runtime::Runtime,
900 blob: &Arc<dyn BlobStore>,
901 prefix: &str,
902 counter: u64,
903 batch: Vec<Pending<M>>,
904 concurrent: bool,
905) -> FlushWork<M> {
906 let (mut work, key, chunks) = prepare_flush_work(prefix, counter, batch);
907 let blob = Arc::clone(blob);
908 let _ = blob.take_media_op_stats();
909 work.put_started = Instant::now();
910 if concurrent {
911 work.put =
912 Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
913 } else {
914 let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
915 let elapsed = work.put_started.elapsed();
916 work.put_result = Some(match put_result {
917 Ok(()) => Ok(elapsed),
918 Err(e) => Err(e),
919 });
920 }
921 work
922}
923
924fn finish_flush_work<S, M>(
925 shared: &Arc<Shared<M>>,
926 blob: &Arc<dyn BlobStore>,
927 sequencer: &Arc<S>,
928 mut work: FlushWork<S::Meta>,
929) -> usize
930where
931 S: Sequencer + 'static,
932 S::Meta: Send + 'static,
933{
934 let release_bytes = work.bytes;
935 let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
938 let put_elapsed = match work.put_result.take().expect("put result is ready") {
939 Ok(elapsed) => elapsed,
940 Err(e) => {
941 send_storage_error(&mut work.responders, e.clone());
942 let mut q = shared.queue.lock().expect("poisoned");
943 let done = work.max_seq;
945 if done >= q.completed_through {
946 q.completed_through = done;
947 }
948 let mut i = 0;
949 while i < q.flush_waiters.len() {
950 if q.flush_waiters[i].0 <= done {
951 let (_, tx) = q.flush_waiters.swap_remove(i);
952 let _ = tx.send(Err(e.clone()));
953 } else {
954 i += 1;
955 }
956 }
957 shared.cv.notify_all();
958 return release_bytes;
959 }
960 };
961
962 let mut media_ops = blob.take_media_op_stats().map(|s| s.media_ops).unwrap_or(1); send_durable_acks(&mut work.responders);
967
968 let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
970 .batch
971 .iter()
972 .zip(work.locations.iter())
973 .map(|(p, loc)| CommitBatch {
974 partition: p.partition.clone(),
975 record_count: p.record_count,
976 location: loc.clone(),
977 meta: &p.meta,
978 })
979 .collect();
980
981 let commit_started = timing.then(Instant::now);
982 let _ = blob.take_media_op_stats();
985 match sequencer.commit(&commit_batches) {
986 Ok(outcomes) => {
987 if let Some(commit_started) = commit_started {
988 eprintln!(
989 "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={}",
990 work.bytes,
991 commit_batches.len(),
992 put_elapsed.as_millis(),
993 commit_started.elapsed().as_millis()
994 );
995 }
996 if let Some(s) = blob.take_media_op_stats() {
997 media_ops = media_ops.saturating_add(s.media_ops);
998 }
999 for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1000 if let Some(tx) = tx.take() {
1001 let (base, last) = match outcome {
1002 CommitOutcome::Assigned {
1003 base_offset,
1004 record_count,
1005 } => (
1006 Some(base_offset),
1007 Some(base_offset + record_count as i64 - 1),
1008 ),
1009 CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1010 };
1011 let _ = tx.send(Ok(AppendOutcome {
1012 base_offset: base,
1013 last_offset: last,
1014 durable: true,
1015 sequenced: true,
1016 }));
1017 }
1018 }
1019 }
1020 Err(e) => {
1021 if let Some(s) = blob.take_media_op_stats() {
1022 media_ops = media_ops.saturating_add(s.media_ops);
1023 }
1024 send_storage_error(&mut work.responders, e);
1025 }
1026 }
1027
1028 {
1029 let mut budget = shared.budget.lock().expect("poisoned");
1030 budget.consume_after_flush(media_ops, Instant::now());
1031 }
1032 {
1033 let mut q = shared.queue.lock().expect("poisoned");
1034 if work.max_seq >= q.completed_through {
1035 q.completed_through = work.max_seq;
1036 }
1037 notify_flush_waiters(&mut q);
1038 }
1039 shared.cv.notify_all();
1040
1041 release_bytes
1042}