1use crate::budget::{
4 BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24 Buffered,
27 Durable,
30 Sequenced,
33}
34
35#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38 pub max_bytes: usize,
43 pub max_batches: usize,
45 pub linger: Duration,
51 pub max_inflight_flushes: usize,
55 pub max_buffered_bytes: usize,
58 pub budget: BudgetConfig,
60}
61
62impl Default for FlushConfig {
63 fn default() -> Self {
64 Self {
65 max_bytes: 1024 * 1024 * 1024,
67 max_batches: 10_000,
68 linger: Duration::from_millis(50),
69 max_inflight_flushes: 1,
71 max_buffered_bytes: 2 * 1024 * 1024 * 1024,
72 budget: BudgetConfig::default(),
73 }
74 }
75}
76
77#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct AppendOutcome {
80 pub base_offset: Option<i64>,
82 pub last_offset: Option<i64>,
84 pub durable: bool,
86 pub sequenced: bool,
88}
89
90#[derive(Clone, Debug, PartialEq, Eq)]
92pub struct FetchedBatch {
93 pub base_offset: i64,
95 pub record_count: i32,
97 pub payload: Bytes,
99}
100
101#[derive(Clone, Debug, PartialEq, Eq)]
103pub struct BufferStats {
104 pub queued_bytes: usize,
106 pub bytes_in_use: usize,
108 pub queued_batches: usize,
110 pub max_buffered_bytes: usize,
112}
113
114type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
115
116fn retryable_storage_error(err: &ObjectLogError) -> bool {
117 matches!(err, ObjectLogError::StorageUnavailable(_))
118}
119
120async fn retry_delay(attempt: usize) {
121 let multiplier = 1u32 << attempt.min(5);
122 tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
123}
124
125async fn put_chunks_with_retries(
126 blob: &Arc<dyn BlobStore>,
127 key: &str,
128 chunks: Vec<Bytes>,
129) -> Result<(), ObjectLogError> {
130 let mut attempt = 0usize;
132 loop {
133 match blob.put_chunks(key, chunks.clone()).await {
134 Ok(()) => return Ok(()),
135 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
136 retry_delay(attempt).await;
137 attempt += 1;
138 }
139 Err(err) => return Err(err),
140 }
141 }
142}
143
144async fn get_range_with_retries(
145 blob: &Arc<dyn BlobStore>,
146 key: &str,
147 range: std::ops::Range<u64>,
148) -> Result<Option<Bytes>, ObjectLogError> {
149 let mut attempt = 0usize;
150 loop {
151 match blob.get_range(key, range.clone()).await {
152 Ok(bytes) => return Ok(bytes),
153 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
154 retry_delay(attempt).await;
155 attempt += 1;
156 }
157 Err(err) => return Err(err),
158 }
159 }
160}
161
162struct Pending<M> {
163 partition: PartitionKey,
164 record_count: i32,
165 payload: Bytes,
166 meta: M,
167 epoch: u64,
169 durability: Durability,
170 responder: Option<Responder>,
171 seq: u64,
173}
174
175struct FlushWork<M> {
176 batch: Vec<Pending<M>>,
177 locations: Vec<BatchLocation>,
178 responders: Vec<(Durability, Option<Responder>)>,
179 bytes: usize,
180 max_seq: u64,
182 data_objects: usize,
184 put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
185 put_started: Instant,
186 put_result: Option<Result<Duration, ObjectLogError>>,
187}
188
189struct CommitJob {
190 task: TokioJoinHandle<usize>,
191 first_seq: u64,
192}
193
194enum TakeBatch<M> {
195 Batch(Vec<Pending<M>>),
196 Empty,
197 Shutdown,
198}
199
200type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
201
202struct Queue<M> {
203 items: VecDeque<Pending<M>>,
204 bytes: usize,
205 bytes_in_use: usize,
206 shutdown: bool,
207 next_seq: u64,
209 completed_through: u64,
211 first_failed: Option<(u64, ObjectLogError)>,
213 force_flush: bool,
215 flush_waiters: Vec<FlushWaiter>,
217 last_enqueue: Option<Instant>,
219 oldest_enqueue: Option<Instant>,
221}
222
223struct Shared<M> {
224 queue: Mutex<Queue<M>>,
225 cv: Condvar,
226 max_buffered_bytes: usize,
227 budget: Mutex<BudgetRuntime>,
229 flush_config: FlushConfig,
231}
232
233pub struct LogEngine<S: Sequencer> {
242 shared: Arc<Shared<S::Meta>>,
243 blob: Arc<dyn BlobStore>,
244 sequencer: Arc<S>,
245 key_prefix: String,
248 writer_id: String,
249 flush_thread: Option<JoinHandle<()>>,
250}
251
252impl<S> LogEngine<S>
253where
254 S: Sequencer + 'static,
255 S::Meta: Send + 'static,
256{
257 pub fn new(
261 blob: Arc<dyn BlobStore>,
262 sequencer: Arc<S>,
263 config: FlushConfig,
264 key_prefix: impl Into<String>,
265 ) -> Self {
266 Self::new_with_writer(blob, sequencer, config, key_prefix, "")
267 }
268
269 pub fn new_with_writer(
273 blob: Arc<dyn BlobStore>,
274 sequencer: Arc<S>,
275 config: FlushConfig,
276 key_prefix: impl Into<String>,
277 writer_id: impl Into<String>,
278 ) -> Self {
279 if let Err(msg) = config.budget.validate() {
280 panic!("invalid FlushConfig.budget: {msg}");
281 }
282 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
283 eprintln!("object-log flush config: {config:?}");
284 }
285 let budget_rt = BudgetRuntime::new(config.budget);
286 let shared = Arc::new(Shared {
287 queue: Mutex::new(Queue {
288 items: VecDeque::new(),
289 bytes: 0,
290 bytes_in_use: 0,
291 shutdown: false,
292 next_seq: 1, completed_through: 0,
294 first_failed: None,
295 force_flush: false,
296 flush_waiters: Vec::new(),
297 last_enqueue: None,
298 oldest_enqueue: None,
299 }),
300 cv: Condvar::new(),
301 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
302 budget: Mutex::new(budget_rt),
303 flush_config: config,
304 });
305 let key_prefix = key_prefix.into();
306 let writer_id = writer_id.into();
307 let flush_thread = {
308 let shared = Arc::clone(&shared);
309 let blob = Arc::clone(&blob);
310 let sequencer = Arc::clone(&sequencer);
311 let prefix = key_prefix.clone();
312 let writer_id = writer_id.clone();
313 std::thread::Builder::new()
314 .name("object-log-flush".into())
315 .spawn(move || flush_loop(shared, blob, sequencer, config, prefix, writer_id))
316 .expect("spawn flush thread")
317 };
318 Self {
319 writer_id,
320 shared,
321 blob,
322 sequencer,
323 key_prefix,
324 flush_thread: Some(flush_thread),
325 }
326 }
327
328 pub fn data_prefix(&self) -> &str {
330 &self.key_prefix
331 }
332
333 pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
335 &self.blob
336 }
337
338 pub fn sequencer(&self) -> &Arc<S> {
340 &self.sequencer
341 }
342
343 pub fn writer_id(&self) -> &str {
345 &self.writer_id
346 }
347
348 pub async fn produce(
353 &self,
354 partition: PartitionKey,
355 payload: Bytes,
356 record_count: i32,
357 meta: S::Meta,
358 durability: Durability,
359 ) -> Result<AppendOutcome, ObjectLogError> {
360 self.produce_at_epoch(partition, payload, record_count, meta, durability, 0)
361 .await
362 }
363
364 pub async fn produce_at_epoch(
366 &self,
367 partition: PartitionKey,
368 payload: Bytes,
369 record_count: i32,
370 meta: S::Meta,
371 durability: Durability,
372 epoch: u64,
373 ) -> Result<AppendOutcome, ObjectLogError> {
374 if record_count <= 0 {
375 return Err(ObjectLogError::InvalidBatch(
376 "record_count must be > 0".into(),
377 ));
378 }
379 if matches!(durability, Durability::Buffered) {
380 self.enqueue(Pending {
381 partition,
382 record_count,
383 payload,
384 meta,
385 epoch,
386 durability,
387 responder: None,
388 seq: 0, })?;
390 return Ok(AppendOutcome {
391 base_offset: None,
392 last_offset: None,
393 durable: false,
394 sequenced: false,
395 });
396 }
397 {
399 let mut budget = self.shared.budget.lock().expect("poisoned");
400 if budget.config.enabled
401 && budget.config.mode == BudgetMode::FailClosed
402 && !budget.reserve_for_fail_closed(Instant::now())
403 {
404 return Err(ObjectLogError::BudgetExceeded(
405 "insufficient durable-ops budget to admit produce".into(),
406 ));
407 }
408 }
409 if self.shared.flush_config.budget.enabled
411 && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
412 {
413 let timeout = self.shared.flush_config.budget.admission_timeout;
414 let deadline = Instant::now() + timeout;
415 loop {
416 let mut budget = self.shared.budget.lock().expect("poisoned");
417 if budget.can_admit_now(Instant::now()) {
418 break;
419 }
420 drop(budget);
421 if Instant::now() >= deadline {
422 return Err(ObjectLogError::BudgetExceeded(
423 "timed out waiting for durable-ops budget".into(),
424 ));
425 }
426 std::thread::sleep(Duration::from_millis(1));
427 }
428 }
429 let (tx, rx) = oneshot::channel();
430 self.enqueue(Pending {
431 partition,
432 record_count,
433 payload,
434 meta,
435 epoch,
436 durability,
437 responder: Some(tx),
438 seq: 0,
439 })?;
440 rx.await
441 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
442 }
443
444 pub async fn flush(&self) -> Result<(), ObjectLogError> {
453 let rx = {
454 let mut q = self.shared.queue.lock().expect("poisoned");
455 if q.shutdown {
456 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
457 }
458 if q.next_seq == 1 {
459 return Ok(());
461 }
462 let barrier = q.next_seq - 1;
463 if let Some((sequence, error)) = &q.first_failed {
464 if *sequence <= barrier {
465 return Err(error.clone());
466 }
467 }
468 if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
469 return Ok(());
470 }
471 let (tx, rx) = oneshot::channel();
472 q.force_flush = true;
473 q.flush_waiters.push((barrier, tx));
474 self.shared.cv.notify_all();
475 rx
476 };
477 rx.await
478 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
479 }
480
481 fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
482 let item_len = item.payload.len();
483 let max_buffered_bytes = self.shared.max_buffered_bytes;
484 let shared = Arc::clone(&self.shared);
485 let mut q = shared.queue.lock().expect("poisoned");
486 while !q.shutdown
487 && q.bytes_in_use > 0
488 && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
489 {
490 q = shared.cv.wait(q).expect("poisoned");
491 }
492 if q.shutdown {
493 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
494 }
495 let now = Instant::now();
496 item.seq = q.next_seq;
497 q.next_seq = q.next_seq.saturating_add(1);
498 q.bytes += item_len;
499 q.bytes_in_use += item_len;
500 q.last_enqueue = Some(now);
501 if q.oldest_enqueue.is_none() {
502 q.oldest_enqueue = Some(now);
503 }
504 q.items.push_back(item);
505 shared.cv.notify_all();
506 Ok(())
507 }
508
509 pub async fn fetch(
511 &self,
512 partition: &PartitionKey,
513 offset: i64,
514 max_bytes: usize,
515 ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
516 let entries = self.sequencer.lookup(partition, offset)?;
517 let mut out = Vec::new();
518 let mut total = 0usize;
519 for e in entries {
520 if total >= max_bytes && !out.is_empty() {
521 break;
522 }
523 let batch = self.load_entry(&e).await?;
524 total += batch.payload.len();
525 out.push(batch);
526 }
527 Ok(out)
528 }
529
530 pub async fn fetch_stream<F>(
537 &self,
538 partition: &PartitionKey,
539 offset: i64,
540 mut visit: F,
541 ) -> Result<(), ObjectLogError>
542 where
543 F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
544 {
545 let entries = self.sequencer.lookup(partition, offset)?;
546 for e in entries {
547 visit(self.load_entry(&e).await?)?;
548 }
549 Ok(())
550 }
551
552 async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
553 let start = e.location.byte_start as u64;
554 let end = start + e.location.byte_len as u64;
555 let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
556 .await?
557 .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
558 Ok(FetchedBatch {
559 base_offset: e.base_offset,
560 record_count: e.record_count,
561 payload: bytes,
562 })
563 }
564
565 pub async fn truncate_before(
568 &self,
569 partition: &PartitionKey,
570 offset: i64,
571 ) -> Result<(), ObjectLogError> {
572 let dead = self.sequencer.truncate_before(partition, offset)?;
573 for object_id in dead {
574 self.blob.delete(&object_id).await?;
575 }
576 Ok(())
577 }
578
579 pub async fn reap_orphans(
589 &self,
590 live: &HashSet<String>,
591 ) -> Result<Vec<String>, ObjectLogError> {
592 reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
593 }
594
595 pub fn buffer_stats(&self) -> BufferStats {
597 let q = self.shared.queue.lock().expect("poisoned");
598 BufferStats {
599 queued_bytes: q.bytes,
600 bytes_in_use: q.bytes_in_use,
601 queued_batches: q.items.len(),
602 max_buffered_bytes: self.shared.max_buffered_bytes,
603 }
604 }
605
606 pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
608 let now = Instant::now();
609 let mut budget = self.shared.budget.lock().expect("poisoned");
610 budget.refill(now);
611 let effective = budget.effective_budget_per_sec;
612 let reason = if !budget.config.enabled {
613 EffectiveReason::Disabled
614 } else if budget.config.budget_per_sec_cap.is_some()
615 && budget
616 .config
617 .budget_per_sec_cap
618 .is_some_and(|c| (effective - c).abs() < 1e-9)
619 {
620 EffectiveReason::ConfigCap
621 } else if budget.ongoing_capacity.is_some() {
622 EffectiveReason::Ongoing
623 } else if budget.startup_capacity.is_some() {
624 EffectiveReason::StartupProbe
625 } else {
626 EffectiveReason::DefaultCapacity
627 };
628 let (queued, last_enq) = {
629 let q = self.shared.queue.lock().expect("poisoned");
630 (q.bytes, q.last_enqueue)
631 };
632 let early = budget.allow_early_flush(now, queued, last_enq);
633 let max_linger = self.shared.flush_config.linger;
634 let eff_linger = if !budget.config.enabled {
635 max_linger
636 } else if early || max_linger.is_zero() {
637 Duration::ZERO
638 } else {
639 max_linger
640 };
641 PipelineSnapshot {
642 budget_per_sec: EffectiveKnob {
643 configured: budget.config.budget_per_sec_cap,
644 startup_measured: budget.startup_capacity,
645 ongoing_measured: budget.ongoing_capacity,
646 effective,
647 reason,
648 },
649 effective_linger_ms: EffectiveKnob {
650 configured: Some(max_linger.as_millis() as u64),
651 startup_measured: None,
652 ongoing_measured: None,
653 effective: eff_linger.as_millis() as u64,
654 reason: if early {
655 EffectiveReason::Ongoing
656 } else {
657 EffectiveReason::Configured
658 },
659 },
660 max_linger_ms: max_linger.as_millis() as u64,
661 token_fill_ratio: budget.fill_ratio(),
662 media_ops_total: budget.media_ops_total,
663 overdraft_total: budget.overdraft_total,
664 flushes_total: budget.flushes_total,
665 undersized_deadline_flushes: budget.undersized_deadline_flushes,
666 predicted_media_ops: budget.predicted_media_ops,
667 budget_enabled: budget.config.enabled,
668 budget_mode: budget.config.mode,
669 }
670 }
671}
672
673pub async fn reap_orphans(
679 blob: &dyn BlobStore,
680 data_prefix: &str,
681 live: &HashSet<String>,
682) -> Result<Vec<String>, ObjectLogError> {
683 let keys = blob.list(data_prefix).await?;
684 let mut deleted = Vec::new();
685 for key in keys {
686 if live.contains(&key) {
687 continue;
688 }
689 blob.delete(&key).await?;
690 deleted.push(key);
691 }
692 Ok(deleted)
693}
694
695impl<S: Sequencer> Drop for LogEngine<S> {
696 fn drop(&mut self) {
697 {
698 let mut q = self.shared.queue.lock().expect("poisoned");
699 q.shutdown = true;
700 }
701 self.shared.cv.notify_all();
702 if let Some(t) = self.flush_thread.take() {
703 let _ = t.join();
704 }
705 }
706}
707
708async fn recover_data_object_counter(
714 blob: &dyn BlobStore,
715 prefix: &str,
716 writer_id: &str,
717) -> Result<u64, ObjectLogError> {
718 let keys = blob.list(prefix).await?;
719 let mut max = 0u64;
720 let writer_prefix = format!("{prefix}{writer_id}/");
721 for key in keys {
722 let Some(suffix) = key.strip_prefix(prefix) else {
723 continue;
724 };
725 if writer_id.is_empty() {
726 if let Ok(n) = suffix.parse::<u64>() {
727 max = max.max(n);
728 }
729 continue;
730 }
731 let Some(rest) = key.strip_prefix(&writer_prefix) else {
732 continue;
733 };
734 if let Some(n) = rest
735 .rsplit('/')
736 .next()
737 .and_then(|tail| tail.parse::<u64>().ok())
738 {
739 max = max.max(n);
740 }
741 }
742 Ok(max)
743}
744
745fn data_object_key(prefix: &str, writer_id: &str, epoch: u64, counter: u64) -> String {
746 if writer_id.is_empty() {
747 format!("{prefix}{counter:020}")
748 } else {
749 format!("{prefix}{writer_id}/{epoch:020}/{counter:020}")
750 }
751}
752
753fn flush_loop<S>(
754 shared: Arc<Shared<S::Meta>>,
755 blob: Arc<dyn BlobStore>,
756 sequencer: Arc<S>,
757 config: FlushConfig,
758 prefix: String,
759 writer_id: String,
760) where
761 S: Sequencer + 'static,
762 S::Meta: Send + 'static,
763{
764 let max_inflight = config.max_inflight_flushes.max(1);
765 let commit_group_limit = if sequencer.supports_multi_object_commit() {
766 max_inflight
767 } else {
768 1
769 };
770 let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
771 .ok()
772 .and_then(|v| v.parse::<usize>().ok())
773 .filter(|v| *v > 0)
774 .unwrap_or_else(|| max_inflight.min(8));
775 let rt = tokio::runtime::Builder::new_multi_thread()
776 .worker_threads(worker_threads)
777 .enable_all()
778 .build()
779 .expect("flush runtime");
780 let mut counter = match rt.block_on(recover_data_object_counter(
784 blob.as_ref(),
785 &prefix,
786 &writer_id,
787 )) {
788 Ok(counter) => counter,
789 Err(error) => {
790 let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
793 return;
794 }
795 };
796 let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
797 let mut active_puts = 0usize;
798 let mut committing: Option<CommitJob> = None;
799 let mut shutdown = false;
800
801 loop {
802 while !shutdown && active_puts < max_inflight {
803 let wait_for_more = pending.is_empty() && committing.is_none();
806 match take_batch(&shared, config, wait_for_more) {
807 TakeBatch::Batch(batch) => {
808 counter += 1;
809 let concurrent = max_inflight > 1;
810 let work = start_flush_work(
811 &rt, &blob, &prefix, &writer_id, counter, batch, concurrent,
812 );
813 if concurrent {
814 active_puts += 1;
815 }
816 pending.push_back(work);
817 }
818 TakeBatch::Empty => break,
819 TakeBatch::Shutdown => shutdown = true,
820 }
821 }
822
823 let mut made_progress = false;
824 for work in pending.iter_mut() {
825 let Some(put) = work.put.as_ref() else {
826 continue;
827 };
828 if !put.is_finished() {
829 continue;
830 }
831 let put = work.put.take().expect("put handle exists");
832 let elapsed = work.put_started.elapsed();
833 work.put_result = Some(match rt.block_on(put) {
834 Ok(Ok(())) => Ok(elapsed),
835 Ok(Err(e)) => Err(e),
836 Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
837 "flush task failed: {e}"
838 ))),
839 });
840 active_puts = active_puts.saturating_sub(1);
841 made_progress = true;
842 }
843
844 if committing
845 .as_ref()
846 .is_some_and(|job| job.task.is_finished())
847 {
848 let job = committing.take().expect("completed commit exists");
849 match rt.block_on(job.task) {
850 Ok(released) => {
851 let mut q = shared.queue.lock().expect("poisoned");
852 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
853 shared.cv.notify_all();
854 }
855 Err(error) => {
856 let uploads = fail_engine_and_take_pending_uploads(
857 &shared,
858 &mut pending,
859 job.first_seq,
860 ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
861 );
862 for upload in uploads {
866 let _ = rt.block_on(upload);
867 }
868 return;
869 }
870 }
871 made_progress = true;
872 }
873
874 if committing.is_none() {
875 if let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit) {
876 if max_inflight == 1 {
877 let released = finish_flush_work(&shared, &blob, &sequencer, work);
879 let mut q = shared.queue.lock().expect("poisoned");
880 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
881 shared.cv.notify_all();
882 } else {
883 let first_seq = work
886 .batch
887 .iter()
888 .map(|p| p.seq)
889 .min()
890 .unwrap_or(work.max_seq);
891 let shared = shared.clone();
892 let blob = blob.clone();
893 let sequencer = sequencer.clone();
894 committing = Some(CommitJob {
895 task: rt.spawn_blocking(move || {
896 finish_flush_work(&shared, &blob, &sequencer, work)
897 }),
898 first_seq,
899 });
900 }
901 made_progress = true;
902 }
903 }
904
905 if pending.is_empty() && committing.is_none() {
906 {
907 let mut q = shared.queue.lock().expect("poisoned");
908 if q.items.is_empty() && q.bytes_in_use == 0 {
909 notify_flush_waiters(&mut q);
912 }
913 }
914 shared.cv.notify_all();
915 if shutdown {
916 let mut q = shared.queue.lock().expect("poisoned");
918 for (_, tx) in q.flush_waiters.drain(..) {
919 let _ = tx.send(Err(ObjectLogError::Sequencer(
920 "engine shutting down".into(),
921 )));
922 }
923 return;
924 }
925 continue;
926 }
927
928 if !made_progress {
929 std::thread::sleep(Duration::from_millis(1));
930 }
931 }
932}
933
934fn fail_engine_and_take_pending_uploads<M>(
937 shared: &Arc<Shared<M>>,
938 pending: &mut VecDeque<FlushWork<M>>,
939 first_seq: u64,
940 error: ObjectLogError,
941) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
942 let mut uploads = Vec::new();
943 for mut work in pending.drain(..) {
944 if let Some(put) = work.put.take() {
945 uploads.push(put);
946 }
947 send_storage_error(&mut work.responders, error.clone());
948 }
949 let mut q = shared
951 .queue
952 .lock()
953 .unwrap_or_else(|poisoned| poisoned.into_inner());
954 q.shutdown = true;
955 q.first_failed.get_or_insert((first_seq, error.clone()));
956 for mut item in q.items.drain(..) {
957 if let Some(tx) = item.responder.take() {
958 let _ = tx.send(Err(error.clone()));
959 }
960 }
961 q.bytes = 0;
962 for (_, tx) in q.flush_waiters.drain(..) {
965 let _ = tx.send(Err(error.clone()));
966 }
967 q.force_flush = false;
968 shared.cv.notify_all();
969 uploads
970}
971
972fn take_ready_commit_group<M>(
975 pending: &mut VecDeque<FlushWork<M>>,
976 max_objects: usize,
977) -> Option<FlushWork<M>> {
978 pending.front()?.put_result.as_ref()?;
979 let mut group = pending.pop_front().expect("ready front exists");
980 if group.put_result.as_ref().is_some_and(Result::is_err) {
981 return Some(group);
982 }
983 while group.data_objects < max_objects.max(1) {
984 let Some(next) = pending.front() else { break };
985 if !next.put_result.as_ref().is_some_and(Result::is_ok)
986 || group.data_objects.saturating_add(next.data_objects) > max_objects
987 {
988 break;
989 }
990 let mut next = pending.pop_front().expect("ready successor exists");
991 group.batch.append(&mut next.batch);
992 group.locations.append(&mut next.locations);
993 group.responders.append(&mut next.responders);
994 group.bytes = group
995 .bytes
996 .checked_add(next.bytes)
997 .expect("bounded pending bytes");
998 group.max_seq = group.max_seq.max(next.max_seq);
999 group.data_objects += next.data_objects;
1000 let next_elapsed = next.put_result.take().unwrap().unwrap();
1001 let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
1002 *elapsed = (*elapsed).max(next_elapsed);
1003 }
1004 Some(group)
1005}
1006
1007fn effective_linger<M>(
1011 shared: &Shared<M>,
1012 config: FlushConfig,
1013 queued_bytes: usize,
1014 last_enqueue: Option<Instant>,
1015 oldest_enqueue: Option<Instant>,
1016) -> Duration {
1017 if config.linger.is_zero() {
1018 return Duration::ZERO;
1019 }
1020 let now = Instant::now();
1021 if config.budget.enabled {
1022 let mut budget = shared.budget.lock().expect("poisoned");
1023 budget.refill(now);
1024 if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
1025 budget.note_early_flush(now);
1026 return Duration::ZERO;
1027 }
1028 }
1029 if let Some(oldest) = oldest_enqueue {
1031 let deadline = oldest + config.linger;
1032 if now >= deadline {
1033 return Duration::ZERO;
1034 }
1035 let until_deadline = deadline.saturating_duration_since(now);
1036 if config.budget.enabled
1038 && !config.budget.early_flush_idle.is_zero()
1039 && let Some(last) = last_enqueue
1040 {
1041 let idle_at = last + config.budget.early_flush_idle;
1042 if idle_at > now && idle_at < deadline {
1043 return idle_at.saturating_duration_since(now);
1044 }
1045 }
1046 return until_deadline;
1047 }
1048 config.linger
1049}
1050
1051fn take_batch<M>(
1052 shared: &Arc<Shared<M>>,
1053 config: FlushConfig,
1054 wait_when_empty: bool,
1055) -> TakeBatch<M> {
1056 let mut q = shared.queue.lock().expect("poisoned");
1057 loop {
1058 if q.items.is_empty() {
1059 if q.shutdown {
1060 return TakeBatch::Shutdown;
1061 }
1062 if !wait_when_empty {
1063 return TakeBatch::Empty;
1064 }
1065 q = shared.cv.wait(q).expect("poisoned");
1069 continue;
1070 }
1071
1072 let linger = {
1073 let queued = q.bytes;
1074 let last_enq = q.last_enqueue;
1075 let oldest = q.oldest_enqueue;
1076 drop(q);
1077 let l = effective_linger(shared, config, queued, last_enq, oldest);
1078 q = shared.queue.lock().expect("poisoned");
1079 l
1080 };
1081
1082 let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1083 let force = q.force_flush;
1084 let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1085 if triggered {
1086 if !size_trigger && !q.shutdown && !q.items.is_empty() {
1087 let mut budget = shared.budget.lock().expect("poisoned");
1088 if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1089 budget.undersized_deadline_flushes += 1;
1090 }
1091 }
1092 break;
1093 }
1094
1095 let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1098 q = guard;
1099 }
1100
1101 let force_drain = q.force_flush;
1102 let mut items = Vec::new();
1103 let mut bytes = 0usize;
1104 while let Some(item) = q.items.pop_front() {
1107 q.bytes = q.bytes.saturating_sub(item.payload.len());
1108 bytes += item.payload.len();
1109 items.push(item);
1110 if bytes >= config.max_bytes {
1111 break;
1112 }
1113 if !force_drain && items.len() >= config.max_batches {
1114 break;
1115 }
1116 }
1117 q.oldest_enqueue = if q.items.is_empty() {
1118 None
1119 } else {
1120 Some(Instant::now())
1122 };
1123 if force_drain {
1125 q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1126 }
1127 if items.is_empty() {
1128 TakeBatch::Empty
1129 } else {
1130 TakeBatch::Batch(items)
1131 }
1132}
1133
1134fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1135 let done = q.completed_through;
1136 let mut i = 0;
1137 while i < q.flush_waiters.len() {
1138 let barrier = q.flush_waiters[i].0;
1139 let failure = q
1140 .first_failed
1141 .as_ref()
1142 .filter(|(sequence, _)| *sequence <= barrier)
1143 .map(|(_, error)| error.clone());
1144 if barrier <= done || failure.is_some() {
1145 let (_, tx) = q.flush_waiters.swap_remove(i);
1146 let _ = tx.send(failure.map_or(Ok(()), Err));
1147 } else {
1148 i += 1;
1149 }
1150 }
1151 if q.flush_waiters.is_empty() {
1152 q.force_flush = false;
1153 } else if q.items.is_empty() && q.bytes_in_use == 0 {
1154 q.force_flush = true;
1156 } else {
1157 q.force_flush = true;
1158 }
1159}
1160
1161fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1162 for (_, tx) in responders.iter_mut() {
1163 if let Some(tx) = tx.take() {
1164 let _ = tx.send(Err(err.clone()));
1165 }
1166 }
1167}
1168
1169fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1170 for (durability, tx) in responders.iter_mut() {
1171 if *durability == Durability::Durable
1172 && let Some(tx) = tx.take()
1173 {
1174 let _ = tx.send(Ok(AppendOutcome {
1175 base_offset: None,
1176 last_offset: None,
1177 durable: true,
1178 sequenced: false,
1179 }));
1180 }
1181 }
1182}
1183
1184fn prepare_flush_work<M>(
1185 prefix: &str,
1186 writer_id: &str,
1187 counter: u64,
1188 mut batch: Vec<Pending<M>>,
1189) -> (FlushWork<M>, String, Vec<Bytes>) {
1190 let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1191 let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1192 let epoch = batch.first().map(|pending| pending.epoch).unwrap_or(0);
1193 let key = data_object_key(prefix, writer_id, epoch, counter);
1194 let mut offset = 0usize;
1195 for p in &batch {
1196 let start = offset as u32;
1197 offset += p.payload.len();
1198 chunks.push(p.payload.clone());
1199 locations.push(BatchLocation {
1200 object_id: key.clone(),
1201 byte_start: start,
1202 byte_len: p.payload.len() as u32,
1203 });
1204 }
1205 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1206 eprintln!(
1207 "object-log seal: key={key} batches={} bytes={offset}",
1208 batch.len()
1209 );
1210 }
1211 let responders: Vec<(Durability, Option<Responder>)> = batch
1212 .iter_mut()
1213 .map(|p| (p.durability, p.responder.take()))
1214 .collect();
1215 let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1216 let work = FlushWork {
1217 batch,
1218 locations,
1219 responders,
1220 bytes: offset,
1221 max_seq,
1222 data_objects: 1,
1223 put: None,
1224 put_started: Instant::now(),
1225 put_result: None,
1226 };
1227 (work, key, chunks)
1228}
1229
1230fn start_flush_work<M>(
1236 rt: &tokio::runtime::Runtime,
1237 blob: &Arc<dyn BlobStore>,
1238 prefix: &str,
1239 writer_id: &str,
1240 counter: u64,
1241 batch: Vec<Pending<M>>,
1242 concurrent: bool,
1243) -> FlushWork<M> {
1244 let (mut work, key, chunks) = prepare_flush_work(prefix, writer_id, counter, batch);
1245 let blob = Arc::clone(blob);
1246 let _ = blob.take_media_op_stats();
1247 work.put_started = Instant::now();
1248 if concurrent {
1249 work.put =
1250 Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
1251 } else {
1252 let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1253 let elapsed = work.put_started.elapsed();
1254 work.put_result = Some(match put_result {
1255 Ok(()) => Ok(elapsed),
1256 Err(e) => Err(e),
1257 });
1258 }
1259 work
1260}
1261
1262fn finish_flush_work<S, M>(
1263 shared: &Arc<Shared<M>>,
1264 blob: &Arc<dyn BlobStore>,
1265 sequencer: &Arc<S>,
1266 mut work: FlushWork<S::Meta>,
1267) -> usize
1268where
1269 S: Sequencer + 'static,
1270 S::Meta: Send + 'static,
1271{
1272 let release_bytes = work.bytes;
1273 let first_seq = work
1274 .batch
1275 .iter()
1276 .map(|pending| pending.seq)
1277 .min()
1278 .unwrap_or(work.max_seq);
1279 let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1283 let put_elapsed = match work.put_result.take().expect("put result is ready") {
1284 Ok(elapsed) => elapsed,
1285 Err(e) => {
1286 send_storage_error(&mut work.responders, e.clone());
1287 let mut q = shared.queue.lock().expect("poisoned");
1288 let done = work.max_seq;
1290 if done >= q.completed_through {
1291 q.completed_through = done;
1292 }
1293 q.first_failed.get_or_insert((first_seq, e));
1294 notify_flush_waiters(&mut q);
1295 shared.cv.notify_all();
1296 return release_bytes;
1297 }
1298 };
1299
1300 let mut media_ops = blob
1302 .take_media_op_stats()
1303 .map(|s| s.media_ops)
1304 .unwrap_or(work.data_objects as u64); send_durable_acks(&mut work.responders);
1308
1309 let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1311 .batch
1312 .iter()
1313 .zip(work.locations.iter())
1314 .map(|(p, loc)| CommitBatch {
1315 partition: p.partition.clone(),
1316 record_count: p.record_count,
1317 location: loc.clone(),
1318 meta: &p.meta,
1319 epoch: p.epoch,
1320 })
1321 .collect();
1322
1323 let commit_started = timing.then(Instant::now);
1324 let _ = blob.take_media_op_stats();
1327 let mut commit_error = None;
1328 match sequencer.commit(&commit_batches) {
1329 Ok(outcomes) => {
1330 if let Some(commit_started) = commit_started {
1331 eprintln!(
1332 "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1333 work.bytes,
1334 commit_batches.len(),
1335 put_elapsed.as_millis(),
1336 commit_started.elapsed().as_millis(),
1337 work.data_objects
1338 );
1339 }
1340 if let Some(s) = blob.take_media_op_stats() {
1341 media_ops = media_ops.saturating_add(s.media_ops);
1342 }
1343 for ((outcome, (_, tx)), pending) in outcomes
1344 .into_iter()
1345 .zip(work.responders.iter_mut())
1346 .zip(work.batch.iter())
1347 {
1348 if let Some(tx) = tx.take() {
1349 let (base, last) = match outcome {
1350 CommitOutcome::Assigned {
1351 base_offset,
1352 record_count,
1353 } => (
1354 Some(base_offset),
1355 Some(base_offset + record_count as i64 - 1),
1356 ),
1357 CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1358 CommitOutcome::Rejected { reason } => {
1359 let error = match sequencer.partition_epoch(&pending.partition) {
1362 Some(current) if current > pending.epoch => {
1363 ObjectLogError::Fenced {
1364 partition: pending.partition.as_str().to_owned(),
1365 epoch: pending.epoch,
1366 current,
1367 }
1368 }
1369 _ => ObjectLogError::Sequencer(reason),
1370 };
1371 let _ = tx.send(Err(error));
1372 continue;
1373 }
1374 };
1375 let _ = tx.send(Ok(AppendOutcome {
1376 base_offset: base,
1377 last_offset: last,
1378 durable: true,
1379 sequenced: true,
1380 }));
1381 }
1382 }
1383 }
1384 Err(e) => {
1385 if let Some(s) = blob.take_media_op_stats() {
1386 media_ops = media_ops.saturating_add(s.media_ops);
1387 }
1388 send_storage_error(&mut work.responders, e.clone());
1389 commit_error = Some(e);
1390 }
1391 }
1392
1393 {
1394 let mut budget = shared.budget.lock().expect("poisoned");
1395 budget.consume_after_flush(media_ops, Instant::now());
1396 }
1397 {
1398 let mut q = shared.queue.lock().expect("poisoned");
1399 if let Some(error) = commit_error {
1400 q.first_failed.get_or_insert((first_seq, error));
1401 }
1402 if work.max_seq >= q.completed_through {
1403 q.completed_through = work.max_seq;
1404 }
1405 notify_flush_waiters(&mut q);
1406 }
1407 shared.cv.notify_all();
1408
1409 release_bytes
1410}
1411
1412#[cfg(test)]
1413mod flush_barrier_tests {
1414 use super::*;
1415
1416 #[test]
1417 fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1418 let error = ObjectLogError::Sequencer("manifest failed".into());
1419 let mut queue = Queue::<()> {
1420 items: VecDeque::new(),
1421 bytes: 0,
1422 bytes_in_use: 1,
1423 shutdown: false,
1424 next_seq: 4,
1425 completed_through: 2,
1426 first_failed: Some((2, error.clone())),
1427 force_flush: true,
1428 flush_waiters: Vec::new(),
1429 last_enqueue: None,
1430 oldest_enqueue: None,
1431 };
1432 let mut receivers = Vec::new();
1433 for barrier in [1, 2, 3] {
1434 let (tx, rx) = oneshot::channel();
1435 queue.flush_waiters.push((barrier, tx));
1436 receivers.push(rx);
1437 }
1438 notify_flush_waiters(&mut queue);
1439 assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1440 assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1441 assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1442 assert!(queue.flush_waiters.is_empty());
1443 }
1444}
1445
1446#[cfg(test)]
1447mod ready_commit_group_tests {
1448 use super::*;
1449 use crate::{ManifestSequencer, MemoryBlobStore};
1450 use async_trait::async_trait;
1451 use std::ops::Range;
1452
1453 fn work(
1454 n: u64,
1455 durability: Durability,
1456 ) -> (
1457 FlushWork<()>,
1458 String,
1459 Vec<Bytes>,
1460 oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1461 ) {
1462 let (tx, rx) = oneshot::channel();
1463 let (mut work, key, chunks) = prepare_flush_work(
1464 "data/",
1465 "",
1466 n,
1467 vec![Pending {
1468 partition: PartitionKey("shared".into()),
1469 record_count: 1,
1470 payload: Bytes::from(format!("body-{n}")),
1471 meta: (),
1472 epoch: 0,
1473 durability,
1474 responder: Some(tx),
1475 seq: n,
1476 }],
1477 );
1478 work.put_result = Some(Ok(Duration::from_millis(n)));
1479 (work, key, chunks, rx)
1480 }
1481
1482 #[test]
1483 fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1484 let mut pending = (1..=5)
1485 .map(|n| work(n, Durability::Sequenced).0)
1486 .collect::<VecDeque<_>>();
1487 let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1488 let mut released = 0;
1489 for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1490 let group = take_ready_commit_group(&mut pending, 2).unwrap();
1491 assert_eq!(
1492 group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1493 expected
1494 );
1495 assert_eq!(group.data_objects, expected.len());
1496 assert_eq!(group.responders.len(), expected.len());
1497 assert_eq!(group.max_seq, *expected.last().unwrap());
1498 for (location, n) in group.locations.iter().zip(&expected) {
1499 assert_eq!(location.object_id, format!("data/{n:020}"));
1500 assert_eq!(location.byte_start, 0);
1501 assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1502 }
1503 released += group.bytes;
1504 }
1505 assert!(pending.is_empty());
1506 assert_eq!(released, original_bytes);
1507
1508 let mut first = work(1, Durability::Sequenced).0;
1509 first.put_result = None;
1510 let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1511 assert!(take_ready_commit_group(&mut pending, 8).is_none());
1512 assert_eq!(
1513 pending.len(),
1514 2,
1515 "a later ready PUT cannot pass an unfinished head"
1516 );
1517 pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1518 let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1519 assert_eq!(failed.data_objects, 1);
1520 assert!(failed.put_result.unwrap().is_err());
1521 assert_eq!(pending[0].max_seq, 2);
1522
1523 let mut failed = work(2, Durability::Sequenced).0;
1524 failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1525 let mut pending = VecDeque::from([
1526 work(1, Durability::Sequenced).0,
1527 failed,
1528 work(3, Durability::Sequenced).0,
1529 ]);
1530 assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1531 assert_eq!(
1532 pending.len(),
1533 2,
1534 "a success group cannot cross a failed PUT"
1535 );
1536 let mut pending = VecDeque::from([
1537 work(1, Durability::Sequenced).0,
1538 work(2, Durability::Sequenced).0,
1539 ]);
1540 assert_eq!(
1541 take_ready_commit_group(&mut pending, 1)
1542 .unwrap()
1543 .data_objects,
1544 1
1545 );
1546 assert_eq!(
1547 pending.len(),
1548 1,
1549 "one-object sequencers retain their call boundary"
1550 );
1551 }
1552
1553 struct GatedManifestStore {
1554 inner: MemoryBlobStore,
1555 entered: std::sync::mpsc::SyncSender<()>,
1556 release: Mutex<Option<oneshot::Receiver<()>>>,
1557 fail: bool,
1558 panic_on_commit: bool,
1559 }
1560
1561 #[async_trait]
1562 impl BlobStore for GatedManifestStore {
1563 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1564 if key.starts_with("manifest/") {
1565 let release = self.release.lock().unwrap().take();
1566 if let Some(release) = release {
1567 self.entered.send(()).unwrap();
1568 release.await.unwrap();
1569 assert!(!self.panic_on_commit, "injected commit panic");
1570 if self.fail {
1571 return Err(ObjectLogError::StorageUnavailable(
1572 "manifest failure".into(),
1573 ));
1574 }
1575 }
1576 }
1577 self.inner.put(key, value).await
1578 }
1579 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1580 self.inner.get(key).await
1581 }
1582 async fn get_range(
1583 &self,
1584 key: &str,
1585 range: Range<u64>,
1586 ) -> Result<Option<Bytes>, ObjectLogError> {
1587 self.inner.get_range(key, range).await
1588 }
1589 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1590 self.inner.list(prefix).await
1591 }
1592 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1593 self.inner.delete(key).await
1594 }
1595 fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1596 self.inner.take_media_op_stats()
1597 }
1598 }
1599
1600 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1601 async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1602 for fail in [false, true] {
1603 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1604 let (release_tx, release_rx) = oneshot::channel();
1605 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1606 inner: MemoryBlobStore::new(),
1607 entered: entered_tx,
1608 release: Mutex::new(Some(release_rx)),
1609 fail,
1610 panic_on_commit: false,
1611 });
1612 let sequencer = Arc::new(
1613 ManifestSequencer::open(blob.clone(), "manifest/")
1614 .await
1615 .unwrap(),
1616 );
1617 assert!(sequencer.supports_multi_object_commit());
1618 let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1619 let (mut second, second_key, second_chunks, mut durable_rx) =
1620 work(2, Durability::Durable);
1621 second.batch[0].partition = PartitionKey("other".into());
1622 let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1623 blob.put_chunks(&first_key, first_chunks).await.unwrap();
1624 blob.put_chunks(&second_key, second_chunks).await.unwrap();
1625 blob.put_chunks(&third_key, third_chunks).await.unwrap();
1626 let mut pending = VecDeque::from([first, second, third]);
1627 let group = take_ready_commit_group(&mut pending, 8).unwrap();
1628 let bytes = group.bytes;
1629 assert_eq!(group.data_objects, 3);
1630 let (flush_tx, mut flush_rx) = oneshot::channel();
1631 let mut config = FlushConfig::default();
1632 config.budget.enabled = false;
1633 let shared: Arc<Shared<()>> = Arc::new(Shared {
1634 queue: Mutex::new(Queue {
1635 items: VecDeque::new(),
1636 bytes: 0,
1637 bytes_in_use: bytes,
1638 shutdown: false,
1639 next_seq: 4,
1640 completed_through: 0,
1641 first_failed: None,
1642 force_flush: true,
1643 flush_waiters: vec![(3, flush_tx)],
1644 last_enqueue: None,
1645 oldest_enqueue: None,
1646 }),
1647 cv: Condvar::new(),
1648 max_buffered_bytes: bytes,
1649 budget: Mutex::new(BudgetRuntime::new(config.budget)),
1650 flush_config: config,
1651 });
1652 let worker = {
1653 let shared = shared.clone();
1654 let blob = blob.clone();
1655 let sequencer = sequencer.clone();
1656 std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1657 };
1658 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1659 assert!(matches!(
1660 sequenced_rx.try_recv(),
1661 Err(oneshot::error::TryRecvError::Empty)
1662 ));
1663 assert!(matches!(
1664 third_rx.try_recv(),
1665 Err(oneshot::error::TryRecvError::Empty)
1666 ));
1667 assert!(matches!(
1668 flush_rx.try_recv(),
1669 Err(oneshot::error::TryRecvError::Empty)
1670 ));
1671 let durable = durable_rx.try_recv().unwrap().unwrap();
1672 assert!(durable.durable && !durable.sequenced);
1673 release_tx.send(()).unwrap();
1674 assert_eq!(worker.join().unwrap(), bytes);
1675 if fail {
1676 assert!(sequenced_rx.await.unwrap().is_err());
1677 assert!(third_rx.await.unwrap().is_err());
1678 assert!(flush_rx.await.unwrap().is_err());
1679 assert_eq!(
1680 sequencer
1681 .high_watermark(&PartitionKey("shared".into()))
1682 .unwrap(),
1683 0
1684 );
1685 assert!(blob.list("manifest/").await.unwrap().is_empty());
1686 } else {
1687 assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1688 assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1689 flush_rx.await.unwrap().unwrap();
1690 assert_eq!(blob.list("manifest/").await.unwrap().len(), 3);
1691 }
1692 assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1693 let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1694 .await
1695 .unwrap();
1696 for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1697 let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1698 assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1699 for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1700 assert_eq!(entry.base_offset, offset as i64);
1701 let loc = &entry.location;
1702 assert_eq!(loc.object_id, format!("data/{object:020}"));
1703 let body = blob
1704 .get_range(
1705 &loc.object_id,
1706 loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1707 )
1708 .await
1709 .unwrap()
1710 .unwrap();
1711 assert_eq!(body, format!("body-{object}"));
1712 }
1713 }
1714 }
1715 }
1716
1717 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1718 async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1719 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1720 let (release_tx, release_rx) = oneshot::channel();
1721 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1722 inner: MemoryBlobStore::new(),
1723 entered: entered_tx,
1724 release: Mutex::new(Some(release_rx)),
1725 fail: false,
1726 panic_on_commit: false,
1727 });
1728 let sequencer = Arc::new(
1729 ManifestSequencer::open(blob.clone(), "manifest/")
1730 .await
1731 .unwrap(),
1732 );
1733 let mut config = FlushConfig::default();
1734 config.max_batches = 1;
1735 config.max_inflight_flushes = 4;
1736 config.linger = Duration::ZERO;
1737 config.budget.enabled = false;
1738 let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1739 let first = {
1740 let engine = engine.clone();
1741 tokio::spawn(async move {
1742 engine
1743 .produce(
1744 PartitionKey("p".into()),
1745 Bytes::from_static(b"first"),
1746 1,
1747 (),
1748 Durability::Sequenced,
1749 )
1750 .await
1751 })
1752 };
1753 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1754 for n in 1..4 {
1755 engine
1756 .produce(
1757 PartitionKey("p".into()),
1758 Bytes::from(format!("later-{n}")),
1759 1,
1760 (),
1761 Durability::Buffered,
1762 )
1763 .await
1764 .unwrap();
1765 }
1766 let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1767 loop {
1768 if blob.list("data/").await.unwrap().len() == 4 {
1769 break;
1770 }
1771 tokio::time::sleep(Duration::from_millis(1)).await;
1772 }
1773 })
1774 .await
1775 .is_ok();
1776 let early_ack = first.is_finished();
1777 release_tx.send(()).unwrap();
1779 first.await.unwrap().unwrap();
1780 engine.flush().await.unwrap();
1781 let rows = engine
1782 .fetch(&PartitionKey("p".into()), 0, 1024)
1783 .await
1784 .unwrap();
1785 assert_eq!(
1786 rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
1787 vec![0, 1, 2, 3]
1788 );
1789 drop(engine);
1790 assert!(
1791 !early_ack,
1792 "sequenced acknowledgement preceded manifest durability"
1793 );
1794 assert!(
1795 upload_progress,
1796 "blocked manifest prevented independent data uploads"
1797 );
1798 }
1799
1800 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1801 async fn commit_completion_releases_admission_bytes_without_more_uploads() {
1802 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1803 let sequencer = Arc::new(
1804 ManifestSequencer::open(blob.clone(), "manifest/")
1805 .await
1806 .unwrap(),
1807 );
1808 let mut config = FlushConfig::default();
1809 config.max_bytes = 4;
1810 config.max_buffered_bytes = 4;
1811 config.max_batches = 1;
1812 config.max_inflight_flushes = 4;
1813 config.linger = Duration::ZERO;
1814 config.budget.enabled = false;
1815 let engine = LogEngine::new(blob, sequencer, config, "data/");
1816 engine
1817 .produce(
1818 PartitionKey("p".into()),
1819 Bytes::from_static(b"full"),
1820 1,
1821 (),
1822 Durability::Sequenced,
1823 )
1824 .await
1825 .unwrap();
1826 tokio::time::timeout(Duration::from_secs(5), async {
1827 while engine.buffer_stats().bytes_in_use != 0 {
1828 tokio::time::sleep(Duration::from_millis(1)).await;
1829 }
1830 })
1831 .await
1832 .expect("completed committer stranded admission bytes");
1833 engine
1834 .produce(
1835 PartitionKey("p".into()),
1836 Bytes::from_static(b"next"),
1837 1,
1838 (),
1839 Durability::Sequenced,
1840 )
1841 .await
1842 .unwrap();
1843 engine.flush().await.unwrap();
1844 assert_eq!(
1845 engine
1846 .fetch(&PartitionKey("p".into()), 0, 100)
1847 .await
1848 .unwrap()
1849 .len(),
1850 2
1851 );
1852 }
1853
1854 struct FailureInjectingStore {
1855 inner: Arc<dyn BlobStore>,
1856 second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
1857 second_upload_completed: std::sync::atomic::AtomicBool,
1858 fail_data_listing: std::sync::atomic::AtomicBool,
1859 }
1860
1861 impl FailureInjectingStore {
1862 fn new(inner: Arc<dyn BlobStore>) -> Self {
1863 Self {
1864 inner,
1865 second_upload_gate: Mutex::new(None),
1866 second_upload_completed: std::sync::atomic::AtomicBool::new(false),
1867 fail_data_listing: std::sync::atomic::AtomicBool::new(false),
1868 }
1869 }
1870 }
1871
1872 #[async_trait]
1873 impl BlobStore for FailureInjectingStore {
1874 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1875 let gate = if key == "data/00000000000000000002" {
1876 self.second_upload_gate.lock().unwrap().take()
1877 } else {
1878 None
1879 };
1880 if let Some((entered, release)) = gate {
1881 entered.send(()).unwrap();
1882 release.await.unwrap();
1883 let result = self.inner.put(key, value).await;
1884 if result.is_ok() {
1885 self.second_upload_completed
1886 .store(true, std::sync::atomic::Ordering::SeqCst);
1887 }
1888 return result;
1889 }
1890 self.inner.put(key, value).await
1891 }
1892 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1893 self.inner.get(key).await
1894 }
1895 async fn get_range(
1896 &self,
1897 key: &str,
1898 range: Range<u64>,
1899 ) -> Result<Option<Bytes>, ObjectLogError> {
1900 self.inner.get_range(key, range).await
1901 }
1902 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1903 if prefix == "data/"
1904 && self
1905 .fail_data_listing
1906 .load(std::sync::atomic::Ordering::SeqCst)
1907 {
1908 return Err(ObjectLogError::StorageUnavailable(
1909 "injected object listing failure".into(),
1910 ));
1911 }
1912 self.inner.list(prefix).await
1913 }
1914 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1915 self.inner.delete(key).await
1916 }
1917 }
1918
1919 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1920 async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
1921 let inner = Arc::new(MemoryBlobStore::new());
1922 let key = "data/00000000000000000001";
1923 let original = Bytes::from_static(b"existing immutable object");
1924 inner.put(key, original.clone()).await.unwrap();
1925 let store = Arc::new(FailureInjectingStore::new(inner.clone()));
1926 store
1927 .fail_data_listing
1928 .store(true, std::sync::atomic::Ordering::SeqCst);
1929 let mut config = FlushConfig::default();
1930 config.max_batches = 1;
1931 config.max_inflight_flushes = 4;
1932 config.linger = Duration::ZERO;
1933 config.budget.enabled = false;
1934 let engine = LogEngine::new(
1935 store,
1936 Arc::new(crate::InMemorySequencer::new()),
1937 config,
1938 "data/",
1939 );
1940 let result = tokio::time::timeout(
1941 Duration::from_secs(5),
1942 engine.produce(
1943 PartitionKey("p".into()),
1944 Bytes::from_static(b"must not overwrite"),
1945 1,
1946 (),
1947 Durability::Sequenced,
1948 ),
1949 )
1950 .await;
1951 let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
1952 tokio::task::spawn_blocking(move || drop(engine))
1953 .await
1954 .unwrap();
1955 let retained = inner.get(key).await.unwrap();
1956 assert_eq!(
1957 retained,
1958 Some(original),
1959 "failed listing must preserve sealed bytes"
1960 );
1961 assert!(
1962 result.unwrap().is_err(),
1963 "failed prefix recovery must reject admission"
1964 );
1965 assert!(
1966 flushed.unwrap().is_err(),
1967 "flush must report failed initialization"
1968 );
1969 assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
1970 }
1971
1972 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1973 async fn engines_commit_and_close_independently() {
1974 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1975 let (release_tx, release_rx) = oneshot::channel();
1976 let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1977 inner: MemoryBlobStore::new(),
1978 entered: entered_tx,
1979 release: Mutex::new(Some(release_rx)),
1980 fail: false,
1981 panic_on_commit: false,
1982 })));
1983 let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
1984 let mut config = FlushConfig::default();
1985 config.max_batches = 1;
1986 config.max_inflight_flushes = 4;
1987 config.linger = Duration::ZERO;
1988 config.budget.enabled = false;
1989 let a_seq = Arc::new(
1990 ManifestSequencer::open(a_blob.clone(), "manifest/")
1991 .await
1992 .unwrap(),
1993 );
1994 let b_seq = Arc::new(
1995 ManifestSequencer::open(b_blob.clone(), "manifest/")
1996 .await
1997 .unwrap(),
1998 );
1999 let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
2000 a.produce(
2001 PartitionKey("p".into()),
2002 Bytes::from_static(b"a-first"),
2003 1,
2004 (),
2005 Durability::Buffered,
2006 )
2007 .await
2008 .unwrap();
2009 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2010 let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
2011 let b_result = tokio::time::timeout(
2012 Duration::from_secs(5),
2013 b.produce(
2014 PartitionKey("p".into()),
2015 Bytes::from_static(b"b-first"),
2016 1,
2017 (),
2018 Durability::Sequenced,
2019 ),
2020 )
2021 .await;
2022 let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
2023 let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
2024 release_tx.send(()).unwrap();
2026 if b_closed.is_err() {
2027 closing_b.await.unwrap();
2028 }
2029 assert!(
2030 b_closed.is_ok(),
2031 "closing B waited for A's blocked manifest"
2032 );
2033 assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
2034 a.flush().await.unwrap();
2035 assert_eq!(
2036 a.produce(
2037 PartitionKey("p".into()),
2038 Bytes::from_static(b"a-second"),
2039 1,
2040 (),
2041 Durability::Sequenced
2042 )
2043 .await
2044 .unwrap()
2045 .base_offset,
2046 Some(1)
2047 );
2048 let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
2049 assert_eq!(
2050 rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
2051 vec![
2052 Bytes::from_static(b"a-first"),
2053 Bytes::from_static(b"a-second")
2054 ]
2055 );
2056 tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
2057 }
2058
2059 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2060 async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
2061 let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
2062 let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
2063 let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
2064 let (upload_release_tx, upload_release_rx) = oneshot::channel();
2065 let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2066 inner: MemoryBlobStore::new(),
2067 entered: manifest_entered_tx,
2068 release: Mutex::new(Some(manifest_release_rx)),
2069 fail: false,
2070 panic_on_commit: true,
2071 })));
2072 *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
2073 let mut config = FlushConfig::default();
2074 config.max_batches = 1;
2075 config.max_inflight_flushes = 4;
2076 config.linger = Duration::ZERO;
2077 config.budget.enabled = false;
2078 let sequencer = Arc::new(
2079 ManifestSequencer::open(store.clone(), "manifest/")
2080 .await
2081 .unwrap(),
2082 );
2083 let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
2084 engine
2085 .produce(
2086 PartitionKey("p".into()),
2087 Bytes::from_static(b"first"),
2088 1,
2089 (),
2090 Durability::Buffered,
2091 )
2092 .await
2093 .unwrap();
2094 manifest_entered_rx
2095 .recv_timeout(Duration::from_secs(5))
2096 .unwrap();
2097 engine
2098 .produce(
2099 PartitionKey("p".into()),
2100 Bytes::from_static(b"second"),
2101 1,
2102 (),
2103 Durability::Buffered,
2104 )
2105 .await
2106 .unwrap();
2107 upload_entered_rx
2108 .recv_timeout(Duration::from_secs(5))
2109 .unwrap();
2110 manifest_release_tx.send(()).unwrap();
2111 tokio::time::timeout(Duration::from_secs(5), async {
2112 while !engine.shared.queue.lock().unwrap().shutdown {
2113 tokio::time::sleep(Duration::from_millis(1)).await;
2114 }
2115 })
2116 .await
2117 .unwrap();
2118 let flush_failed = engine.flush().await.is_err();
2119 let upload_was_owned = upload_release_tx.send(()).is_ok();
2122 tokio::time::timeout(
2123 Duration::from_secs(5),
2124 tokio::task::spawn_blocking(move || drop(engine)),
2125 )
2126 .await
2127 .unwrap()
2128 .unwrap();
2129 assert!(flush_failed);
2130 assert!(upload_was_owned);
2131 assert!(
2132 store
2133 .second_upload_completed
2134 .load(std::sync::atomic::Ordering::SeqCst)
2135 );
2136 assert_eq!(store.list("data/").await.unwrap().len(), 2);
2137 let sequencer = Arc::new(
2138 ManifestSequencer::open(store.clone(), "manifest/")
2139 .await
2140 .unwrap(),
2141 );
2142 let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2143 assert!(
2144 reopened
2145 .fetch(&PartitionKey("p".into()), 0, 100)
2146 .await
2147 .unwrap()
2148 .is_empty()
2149 );
2150 assert_eq!(
2151 reopened
2152 .produce(
2153 PartitionKey("p".into()),
2154 Bytes::from_static(b"after-reopen"),
2155 1,
2156 (),
2157 Durability::Sequenced
2158 )
2159 .await
2160 .unwrap()
2161 .base_offset,
2162 Some(0)
2163 );
2164 let rows = reopened
2165 .fetch(&PartitionKey("p".into()), 0, 100)
2166 .await
2167 .unwrap();
2168 assert_eq!(rows.len(), 1);
2169 assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2170 assert_eq!(store.list("data/").await.unwrap().len(), 3);
2171 tokio::task::spawn_blocking(move || drop(reopened))
2172 .await
2173 .unwrap();
2174 }
2175
2176 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2177 async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2178 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2179 let (release_tx, release_rx) = oneshot::channel();
2180 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2181 inner: MemoryBlobStore::new(),
2182 entered: entered_tx,
2183 release: Mutex::new(Some(release_rx)),
2184 fail: false,
2185 panic_on_commit: false,
2186 });
2187 let sequencer = Arc::new(
2188 ManifestSequencer::open(blob.clone(), "manifest/")
2189 .await
2190 .unwrap(),
2191 );
2192 let mut config = FlushConfig::default();
2193 config.max_batches = 1;
2194 config.max_inflight_flushes = 4;
2195 config.linger = Duration::ZERO;
2196 config.budget.enabled = false;
2197 let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2198 engine
2199 .produce(
2200 PartitionKey("p".into()),
2201 Bytes::from_static(b"first"),
2202 1,
2203 (),
2204 Durability::Buffered,
2205 )
2206 .await
2207 .unwrap();
2208 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2209 engine
2210 .produce(
2211 PartitionKey("p".into()),
2212 Bytes::from_static(b"second"),
2213 1,
2214 (),
2215 Durability::Buffered,
2216 )
2217 .await
2218 .unwrap();
2219 let shared = engine.shared.clone();
2220 let dropping = tokio::task::spawn_blocking(move || drop(engine));
2221 tokio::time::timeout(Duration::from_secs(5), async {
2222 while !shared.queue.lock().unwrap().shutdown {
2223 tokio::time::sleep(Duration::from_millis(1)).await;
2224 }
2225 })
2226 .await
2227 .unwrap();
2228 let premature = dropping.is_finished();
2229 release_tx.send(()).unwrap();
2230 tokio::time::timeout(Duration::from_secs(5), dropping)
2231 .await
2232 .unwrap()
2233 .unwrap();
2234 assert!(!premature, "shutdown returned before manifest durability");
2235 let reopened = Arc::new(
2236 ManifestSequencer::open(blob.clone(), "manifest/")
2237 .await
2238 .unwrap(),
2239 );
2240 let engine = LogEngine::new(blob, reopened, config, "data/");
2241 let rows = engine
2242 .fetch(&PartitionKey("p".into()), 0, 100)
2243 .await
2244 .unwrap();
2245 assert_eq!(
2246 rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2247 vec![b"first".as_slice(), b"second".as_slice()]
2248 );
2249 }
2250
2251 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2252 async fn panicked_committer_closes_admission_and_fails_flush() {
2253 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2254 let (release_tx, release_rx) = oneshot::channel();
2255 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2256 inner: MemoryBlobStore::new(),
2257 entered: entered_tx,
2258 release: Mutex::new(Some(release_rx)),
2259 fail: false,
2260 panic_on_commit: true,
2261 });
2262 let sequencer = Arc::new(
2263 ManifestSequencer::open(blob.clone(), "manifest/")
2264 .await
2265 .unwrap(),
2266 );
2267 let mut config = FlushConfig::default();
2268 config.max_batches = 1;
2269 config.max_inflight_flushes = 4;
2270 config.linger = Duration::ZERO;
2271 config.budget.enabled = false;
2272 let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2273 let first = {
2274 let engine = engine.clone();
2275 tokio::spawn(async move {
2276 engine
2277 .produce(
2278 PartitionKey("p".into()),
2279 Bytes::from_static(b"first"),
2280 1,
2281 (),
2282 Durability::Sequenced,
2283 )
2284 .await
2285 })
2286 };
2287 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2288 engine
2289 .produce(
2290 PartitionKey("p".into()),
2291 Bytes::from_static(b"later"),
2292 1,
2293 (),
2294 Durability::Buffered,
2295 )
2296 .await
2297 .unwrap();
2298 let flushing = {
2299 let engine = engine.clone();
2300 tokio::spawn(async move { engine.flush().await })
2301 };
2302 release_tx.send(()).unwrap();
2303 assert!(
2304 tokio::time::timeout(Duration::from_secs(5), first)
2305 .await
2306 .unwrap()
2307 .unwrap()
2308 .is_err()
2309 );
2310 assert!(
2311 tokio::time::timeout(Duration::from_secs(5), flushing)
2312 .await
2313 .unwrap()
2314 .unwrap()
2315 .is_err()
2316 );
2317 tokio::time::timeout(Duration::from_secs(5), async {
2318 while !engine.shared.queue.lock().unwrap().shutdown {
2319 tokio::time::sleep(Duration::from_millis(1)).await;
2320 }
2321 })
2322 .await
2323 .unwrap();
2324 assert!(
2325 engine
2326 .produce(
2327 PartitionKey("p".into()),
2328 Bytes::from_static(b"rejected"),
2329 1,
2330 (),
2331 Durability::Buffered
2332 )
2333 .await
2334 .is_err()
2335 );
2336 assert!(engine.flush().await.is_err());
2337 }
2338 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2339 async fn committed_reads_continue_during_new_manifest_publication() {
2340 for fail in [false, true] {
2341 let memory = MemoryBlobStore::new();
2342 let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2343 let seed_sequencer = Arc::new(
2344 ManifestSequencer::open(seed_blob.clone(), "manifest/")
2345 .await
2346 .unwrap(),
2347 );
2348 let mut config = FlushConfig::default();
2349 config.max_batches = 1;
2350 config.max_inflight_flushes = 4;
2351 config.linger = Duration::ZERO;
2352 config.budget.enabled = false;
2353 let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2354 seed.produce(
2355 PartitionKey("p".into()),
2356 Bytes::from_static(b"committed"),
2357 1,
2358 (),
2359 Durability::Sequenced,
2360 )
2361 .await
2362 .unwrap();
2363 drop(seed);
2364 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2365 let (release_tx, release_rx) = oneshot::channel();
2366 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2367 inner: memory,
2368 entered: entered_tx,
2369 release: Mutex::new(Some(release_rx)),
2370 fail,
2371 panic_on_commit: false,
2372 });
2373 let sequencer = Arc::new(
2374 ManifestSequencer::open(blob.clone(), "manifest/")
2375 .await
2376 .unwrap(),
2377 );
2378 let engine = Arc::new(LogEngine::new(
2379 blob.clone(),
2380 sequencer.clone(),
2381 config,
2382 "data/",
2383 ));
2384 let producing = {
2385 let engine = engine.clone();
2386 tokio::spawn(async move {
2387 engine
2388 .produce(
2389 PartitionKey("p".into()),
2390 Bytes::from_static(b"new"),
2391 1,
2392 (),
2393 Durability::Sequenced,
2394 )
2395 .await
2396 })
2397 };
2398 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2399 let mut reading = tokio::task::spawn_blocking(move || {
2400 let p = PartitionKey("p".into());
2401 (
2402 sequencer.high_watermark(&p).unwrap(),
2403 sequencer.lookup(&p, 0).unwrap(),
2404 sequencer.snapshot(),
2405 )
2406 });
2407 let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2408 let early_ack = producing.is_finished();
2409 release_tx.send(()).unwrap();
2411 let outcome = producing.await.unwrap();
2412 assert_eq!(outcome.is_err(), fail);
2413 let reads_progressed = observed.is_ok();
2414 let (watermark, entries, snapshot) = match observed {
2415 Ok(result) => result.unwrap(),
2416 Err(_) => reading.await.unwrap(),
2417 };
2418 assert!(!early_ack);
2419 assert!(
2420 reads_progressed,
2421 "durable manifest publication blocked committed index reads"
2422 );
2423 assert_eq!(watermark, 1);
2424 assert_eq!(entries.len(), 1);
2425 assert_eq!(snapshot.manifest_count, 1);
2426 let rows = engine
2427 .fetch(&PartitionKey("p".into()), 0, 100)
2428 .await
2429 .unwrap();
2430 assert_eq!(rows.len(), if fail { 1 } else { 2 });
2431 assert_eq!(rows[0].payload.as_ref(), b"committed");
2432 drop(engine);
2433 let reopened = Arc::new(
2434 ManifestSequencer::open(blob.clone(), "manifest/")
2435 .await
2436 .unwrap(),
2437 );
2438 let engine = LogEngine::new(blob, reopened, config, "data/");
2439 assert_eq!(
2440 engine
2441 .fetch(&PartitionKey("p".into()), 0, 100)
2442 .await
2443 .unwrap()
2444 .len(),
2445 if fail { 1 } else { 2 }
2446 );
2447 }
2448 }
2449 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2450 async fn manifest_mutations_remain_serialized_while_readers_progress() {
2451 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2452 let (release_tx, release_rx) = oneshot::channel();
2453 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2454 inner: MemoryBlobStore::new(),
2455 entered: entered_tx,
2456 release: Mutex::new(Some(release_rx)),
2457 fail: false,
2458 panic_on_commit: false,
2459 });
2460 let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2461 let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2462 tokio::task::spawn_blocking(move || {
2463 sequencer.commit(&[CommitBatch {
2464 partition: PartitionKey("p".into()),
2465 record_count: 1,
2466 location: BatchLocation {
2467 object_id: key.into(),
2468 byte_start: 0,
2469 byte_len: 1,
2470 },
2471 meta: &(),
2472 epoch: 0,
2473 }])
2474 })
2475 };
2476 let first = commit(sequencer.clone(), "data/first");
2477 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2478 let second = commit(sequencer.clone(), "data/second");
2479 let truncating = {
2480 let sequencer = sequencer.clone();
2481 tokio::task::spawn_blocking(move || {
2482 sequencer.truncate_before(&PartitionKey("p".into()), 1)
2483 })
2484 };
2485 let reading = {
2486 let sequencer = sequencer.clone();
2487 tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2488 };
2489 let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2490 release_tx.send(()).unwrap();
2491 assert_eq!(
2492 observed.expect("committed reads blocked").unwrap().unwrap(),
2493 0
2494 );
2495 let first = tokio::time::timeout(Duration::from_secs(5), first)
2496 .await
2497 .unwrap()
2498 .unwrap()
2499 .unwrap();
2500 let second = tokio::time::timeout(Duration::from_secs(5), second)
2501 .await
2502 .unwrap()
2503 .unwrap()
2504 .unwrap();
2505 let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2506 .await
2507 .unwrap()
2508 .unwrap()
2509 .unwrap();
2510 assert_eq!(
2511 first,
2512 vec![CommitOutcome::Assigned {
2513 base_offset: 0,
2514 record_count: 1
2515 }]
2516 );
2517 assert_eq!(
2518 second,
2519 vec![CommitOutcome::Assigned {
2520 base_offset: 1,
2521 record_count: 1
2522 }]
2523 );
2524 assert_eq!(dropped, vec!["data/first".to_string()]);
2525 let p = PartitionKey("p".into());
2526 assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2527 assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2528 let entries = sequencer.lookup(&p, 0).unwrap();
2529 assert_eq!(entries.len(), 1);
2530 assert_eq!(entries[0].location.object_id, "data/second");
2531 assert_eq!(sequencer.snapshot().manifest_count, 2);
2532 }
2533}
2534
2535#[cfg(test)]
2536mod idle_batch_wait_tests {
2537 use super::*;
2538 use std::sync::mpsc;
2539
2540 fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2541 Arc::new(Shared {
2542 queue: Mutex::new(Queue {
2543 items: VecDeque::new(),
2544 bytes: 0,
2545 bytes_in_use: 0,
2546 shutdown: false,
2547 next_seq: 1,
2548 completed_through: 0,
2549 first_failed: None,
2550 force_flush: false,
2551 flush_waiters: Vec::new(),
2552 last_enqueue: None,
2553 oldest_enqueue: None,
2554 }),
2555 cv: Condvar::new(),
2556 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2557 budget: Mutex::new(BudgetRuntime::new(config.budget)),
2558 flush_config: config,
2559 })
2560 }
2561
2562 #[test]
2563 fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2564 for linger in [Duration::ZERO, Duration::from_millis(1)] {
2565 for shutdown in [false, true] {
2566 let config = FlushConfig {
2567 linger,
2568 ..FlushConfig::default()
2569 };
2570 let shared = empty_shared(config);
2571 let (started_tx, started_rx) = mpsc::channel();
2572 let (result_tx, result_rx) = mpsc::channel();
2573 let worker_shared = shared.clone();
2574 let worker = std::thread::spawn(move || {
2575 started_tx.send(()).unwrap();
2576 assert!(
2577 result_tx
2578 .send(take_batch(&worker_shared, config, true))
2579 .is_ok()
2580 );
2581 });
2582 started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2583 let early = result_rx.recv_timeout(Duration::from_millis(100));
2584 let returned_while_idle = early.is_ok();
2585 {
2586 let mut q = shared.queue.lock().unwrap();
2587 if shutdown {
2588 q.shutdown = true;
2589 } else {
2590 let payload = Bytes::from_static(b"first-after-idle");
2591 q.bytes = payload.len();
2592 q.bytes_in_use = payload.len();
2593 q.next_seq = 2;
2594 q.last_enqueue = Some(Instant::now());
2595 q.oldest_enqueue = q.last_enqueue;
2596 q.force_flush = true;
2597 q.items.push_back(Pending {
2598 partition: PartitionKey("p".into()),
2599 record_count: 1,
2600 payload,
2601 meta: (),
2602 epoch: 0,
2603 durability: Durability::Buffered,
2604 responder: None,
2605 seq: 1,
2606 });
2607 }
2608 shared.cv.notify_all();
2609 }
2610 let result = early
2611 .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2612 .unwrap();
2613 worker.join().unwrap();
2614 assert!(
2615 !returned_while_idle,
2616 "idle worker polled with linger={linger:?}"
2617 );
2618 match result {
2619 TakeBatch::Shutdown => assert!(shutdown),
2620 TakeBatch::Batch(items) => {
2621 assert!(!shutdown);
2622 assert_eq!(items.len(), 1);
2623 assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2624 assert_eq!(items[0].seq, 1);
2625 }
2626 TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2627 }
2628 }
2629 }
2630 }
2631}