1use crate::budget::{
4 BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24 Buffered,
27 Durable,
30 Sequenced,
33}
34
35#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38 pub max_bytes: usize,
43 pub max_batches: usize,
45 pub linger: Duration,
51 pub max_inflight_flushes: usize,
55 pub max_buffered_bytes: usize,
58 pub budget: BudgetConfig,
60}
61
62impl Default for FlushConfig {
63 fn default() -> Self {
64 Self {
65 max_bytes: 1024 * 1024 * 1024,
67 max_batches: 10_000,
68 linger: Duration::from_millis(50),
69 max_inflight_flushes: 1,
71 max_buffered_bytes: 2 * 1024 * 1024 * 1024,
72 budget: BudgetConfig::default(),
73 }
74 }
75}
76
77#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct AppendOutcome {
80 pub base_offset: Option<i64>,
82 pub last_offset: Option<i64>,
84 pub durable: bool,
86 pub sequenced: bool,
88}
89
90#[derive(Clone, Debug, PartialEq, Eq)]
92pub struct FetchedBatch {
93 pub base_offset: i64,
95 pub record_count: i32,
97 pub payload: Bytes,
99}
100
101#[derive(Clone, Debug, PartialEq, Eq)]
103pub struct BufferStats {
104 pub queued_bytes: usize,
106 pub bytes_in_use: usize,
108 pub queued_batches: usize,
110 pub max_buffered_bytes: usize,
112}
113
114type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
115
116fn retryable_storage_error(err: &ObjectLogError) -> bool {
117 matches!(err, ObjectLogError::StorageUnavailable(_))
118}
119
120async fn retry_delay(attempt: usize) {
121 let multiplier = 1u32 << attempt.min(5);
122 tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
123}
124
125async fn put_chunks_with_retries(
126 blob: &Arc<dyn BlobStore>,
127 key: &str,
128 chunks: Vec<Bytes>,
129) -> Result<(), ObjectLogError> {
130 let mut attempt = 0usize;
132 loop {
133 match blob.put_chunks(key, chunks.clone()).await {
134 Ok(()) => return Ok(()),
135 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
136 retry_delay(attempt).await;
137 attempt += 1;
138 }
139 Err(err) => return Err(err),
140 }
141 }
142}
143
144async fn get_range_with_retries(
145 blob: &Arc<dyn BlobStore>,
146 key: &str,
147 range: std::ops::Range<u64>,
148) -> Result<Option<Bytes>, ObjectLogError> {
149 let mut attempt = 0usize;
150 loop {
151 match blob.get_range(key, range.clone()).await {
152 Ok(bytes) => return Ok(bytes),
153 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
154 retry_delay(attempt).await;
155 attempt += 1;
156 }
157 Err(err) => return Err(err),
158 }
159 }
160}
161
162struct Pending<M> {
163 partition: PartitionKey,
164 record_count: i32,
165 payload: Bytes,
166 meta: M,
167 epoch: u64,
169 durability: Durability,
170 responder: Option<Responder>,
171 seq: u64,
173}
174
175struct FlushWork<M> {
176 batch: Vec<Pending<M>>,
177 locations: Vec<BatchLocation>,
178 responders: Vec<(Durability, Option<Responder>)>,
179 bytes: usize,
180 max_seq: u64,
182 data_objects: usize,
184 put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
185 put_started: Instant,
186 put_result: Option<Result<Duration, ObjectLogError>>,
187}
188
189struct CommitJob {
190 task: TokioJoinHandle<usize>,
191 first_seq: u64,
192}
193
194enum TakeBatch<M> {
195 Batch(Vec<Pending<M>>),
196 Empty,
197 Shutdown,
198}
199
200type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
201
202struct Queue<M> {
203 items: VecDeque<Pending<M>>,
204 bytes: usize,
205 bytes_in_use: usize,
206 shutdown: bool,
207 next_seq: u64,
209 completed_through: u64,
211 first_failed: Option<(u64, ObjectLogError)>,
213 force_flush: bool,
215 flush_waiters: Vec<FlushWaiter>,
217 last_enqueue: Option<Instant>,
219 oldest_enqueue: Option<Instant>,
221}
222
223struct Shared<M> {
224 queue: Mutex<Queue<M>>,
225 cv: Condvar,
226 max_buffered_bytes: usize,
227 budget: Mutex<BudgetRuntime>,
229 flush_config: FlushConfig,
231}
232
233pub struct LogEngine<S: Sequencer> {
242 shared: Arc<Shared<S::Meta>>,
243 blob: Arc<dyn BlobStore>,
244 sequencer: Arc<S>,
245 key_prefix: String,
248 writer_id: String,
249 flush_thread: Option<JoinHandle<()>>,
250}
251
252impl<S> LogEngine<S>
253where
254 S: Sequencer + 'static,
255 S::Meta: Send + 'static,
256{
257 pub fn new(
261 blob: Arc<dyn BlobStore>,
262 sequencer: Arc<S>,
263 config: FlushConfig,
264 key_prefix: impl Into<String>,
265 ) -> Self {
266 Self::new_with_writer(blob, sequencer, config, key_prefix, "")
267 }
268
269 pub fn new_with_writer(
273 blob: Arc<dyn BlobStore>,
274 sequencer: Arc<S>,
275 config: FlushConfig,
276 key_prefix: impl Into<String>,
277 writer_id: impl Into<String>,
278 ) -> Self {
279 if let Err(msg) = config.budget.validate() {
280 panic!("invalid FlushConfig.budget: {msg}");
281 }
282 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
283 eprintln!("object-log flush config: {config:?}");
284 }
285 let budget_rt = BudgetRuntime::new(config.budget);
286 let shared = Arc::new(Shared {
287 queue: Mutex::new(Queue {
288 items: VecDeque::new(),
289 bytes: 0,
290 bytes_in_use: 0,
291 shutdown: false,
292 next_seq: 1, completed_through: 0,
294 first_failed: None,
295 force_flush: false,
296 flush_waiters: Vec::new(),
297 last_enqueue: None,
298 oldest_enqueue: None,
299 }),
300 cv: Condvar::new(),
301 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
302 budget: Mutex::new(budget_rt),
303 flush_config: config,
304 });
305 let key_prefix = key_prefix.into();
306 let writer_id = writer_id.into();
307 let flush_thread = {
308 let shared = Arc::clone(&shared);
309 let blob = Arc::clone(&blob);
310 let sequencer = Arc::clone(&sequencer);
311 let prefix = key_prefix.clone();
312 let writer_id = writer_id.clone();
313 std::thread::Builder::new()
314 .name("object-log-flush".into())
315 .spawn(move || flush_loop(shared, blob, sequencer, config, prefix, writer_id))
316 .expect("spawn flush thread")
317 };
318 Self {
319 writer_id,
320 shared,
321 blob,
322 sequencer,
323 key_prefix,
324 flush_thread: Some(flush_thread),
325 }
326 }
327
328 pub fn data_prefix(&self) -> &str {
330 &self.key_prefix
331 }
332
333 pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
335 &self.blob
336 }
337
338 pub fn sequencer(&self) -> &Arc<S> {
340 &self.sequencer
341 }
342
343 pub fn writer_id(&self) -> &str {
345 &self.writer_id
346 }
347
348 pub async fn produce(
353 &self,
354 partition: PartitionKey,
355 payload: Bytes,
356 record_count: i32,
357 meta: S::Meta,
358 durability: Durability,
359 ) -> Result<AppendOutcome, ObjectLogError> {
360 self.produce_at_epoch(partition, payload, record_count, meta, durability, 0)
361 .await
362 }
363
364 pub async fn produce_at_epoch(
366 &self,
367 partition: PartitionKey,
368 payload: Bytes,
369 record_count: i32,
370 meta: S::Meta,
371 durability: Durability,
372 epoch: u64,
373 ) -> Result<AppendOutcome, ObjectLogError> {
374 if record_count <= 0 {
375 return Err(ObjectLogError::InvalidBatch(
376 "record_count must be > 0".into(),
377 ));
378 }
379 if matches!(durability, Durability::Buffered) {
380 self.enqueue(Pending {
381 partition,
382 record_count,
383 payload,
384 meta,
385 epoch,
386 durability,
387 responder: None,
388 seq: 0, })?;
390 return Ok(AppendOutcome {
391 base_offset: None,
392 last_offset: None,
393 durable: false,
394 sequenced: false,
395 });
396 }
397 {
399 let mut budget = self.shared.budget.lock().expect("poisoned");
400 if budget.config.enabled
401 && budget.config.mode == BudgetMode::FailClosed
402 && !budget.reserve_for_fail_closed(Instant::now())
403 {
404 return Err(ObjectLogError::BudgetExceeded(
405 "insufficient durable-ops budget to admit produce".into(),
406 ));
407 }
408 }
409 if self.shared.flush_config.budget.enabled
411 && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
412 {
413 let timeout = self.shared.flush_config.budget.admission_timeout;
414 let deadline = Instant::now() + timeout;
415 loop {
416 let mut budget = self.shared.budget.lock().expect("poisoned");
417 if budget.can_admit_now(Instant::now()) {
418 break;
419 }
420 drop(budget);
421 if Instant::now() >= deadline {
422 return Err(ObjectLogError::BudgetExceeded(
423 "timed out waiting for durable-ops budget".into(),
424 ));
425 }
426 std::thread::sleep(Duration::from_millis(1));
427 }
428 }
429 let (tx, rx) = oneshot::channel();
430 self.enqueue(Pending {
431 partition,
432 record_count,
433 payload,
434 meta,
435 epoch,
436 durability,
437 responder: Some(tx),
438 seq: 0,
439 })?;
440 rx.await
441 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
442 }
443
444 pub async fn flush(&self) -> Result<(), ObjectLogError> {
453 let rx = {
454 let mut q = self.shared.queue.lock().expect("poisoned");
455 if q.shutdown {
456 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
457 }
458 if q.next_seq == 1 {
459 return Ok(());
461 }
462 let barrier = q.next_seq - 1;
463 if let Some((sequence, error)) = &q.first_failed {
464 if *sequence <= barrier {
465 return Err(error.clone());
466 }
467 }
468 if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
469 return Ok(());
470 }
471 let (tx, rx) = oneshot::channel();
472 q.force_flush = true;
473 q.flush_waiters.push((barrier, tx));
474 self.shared.cv.notify_all();
475 rx
476 };
477 rx.await
478 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
479 }
480
481 fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
482 let item_len = item.payload.len();
483 let max_buffered_bytes = self.shared.max_buffered_bytes;
484 let shared = Arc::clone(&self.shared);
485 let mut q = shared.queue.lock().expect("poisoned");
486 while !q.shutdown
487 && q.bytes_in_use > 0
488 && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
489 {
490 q = shared.cv.wait(q).expect("poisoned");
491 }
492 if q.shutdown {
493 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
494 }
495 let now = Instant::now();
496 item.seq = q.next_seq;
497 q.next_seq = q.next_seq.saturating_add(1);
498 q.bytes += item_len;
499 q.bytes_in_use += item_len;
500 q.last_enqueue = Some(now);
501 if q.oldest_enqueue.is_none() {
502 q.oldest_enqueue = Some(now);
503 }
504 q.items.push_back(item);
505 shared.cv.notify_all();
506 Ok(())
507 }
508
509 pub async fn fetch(
511 &self,
512 partition: &PartitionKey,
513 offset: i64,
514 max_bytes: usize,
515 ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
516 let entries = self.sequencer.lookup(partition, offset)?;
517 let mut out = Vec::new();
518 let mut total = 0usize;
519 for e in entries {
520 if total >= max_bytes && !out.is_empty() {
521 break;
522 }
523 let batch = self.load_entry(&e).await?;
524 total += batch.payload.len();
525 out.push(batch);
526 }
527 Ok(out)
528 }
529
530 pub async fn fetch_stream<F>(
537 &self,
538 partition: &PartitionKey,
539 offset: i64,
540 mut visit: F,
541 ) -> Result<(), ObjectLogError>
542 where
543 F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
544 {
545 let entries = self.sequencer.lookup(partition, offset)?;
546 for e in entries {
547 visit(self.load_entry(&e).await?)?;
548 }
549 Ok(())
550 }
551
552 async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
553 let start = e.location.byte_start as u64;
554 let end = start + e.location.byte_len as u64;
555 let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
556 .await?
557 .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
558 Ok(FetchedBatch {
559 base_offset: e.base_offset,
560 record_count: e.record_count,
561 payload: bytes,
562 })
563 }
564
565 pub async fn truncate_before(
568 &self,
569 partition: &PartitionKey,
570 offset: i64,
571 ) -> Result<(), ObjectLogError> {
572 let dead = self.sequencer.truncate_before(partition, offset)?;
573 for object_id in dead {
574 self.blob.delete(&object_id).await?;
575 }
576 Ok(())
577 }
578
579 pub async fn reap_orphans(
589 &self,
590 live: &HashSet<String>,
591 ) -> Result<Vec<String>, ObjectLogError> {
592 reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
593 }
594
595 pub fn buffer_stats(&self) -> BufferStats {
597 let q = self.shared.queue.lock().expect("poisoned");
598 BufferStats {
599 queued_bytes: q.bytes,
600 bytes_in_use: q.bytes_in_use,
601 queued_batches: q.items.len(),
602 max_buffered_bytes: self.shared.max_buffered_bytes,
603 }
604 }
605
606 pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
608 let now = Instant::now();
609 let mut budget = self.shared.budget.lock().expect("poisoned");
610 budget.refill(now);
611 let effective = budget.effective_budget_per_sec;
612 let reason = if !budget.config.enabled {
613 EffectiveReason::Disabled
614 } else if budget.config.budget_per_sec_cap.is_some()
615 && budget
616 .config
617 .budget_per_sec_cap
618 .is_some_and(|c| (effective - c).abs() < 1e-9)
619 {
620 EffectiveReason::ConfigCap
621 } else if budget.ongoing_capacity.is_some() {
622 EffectiveReason::Ongoing
623 } else if budget.startup_capacity.is_some() {
624 EffectiveReason::StartupProbe
625 } else {
626 EffectiveReason::DefaultCapacity
627 };
628 let (queued, last_enq) = {
629 let q = self.shared.queue.lock().expect("poisoned");
630 (q.bytes, q.last_enqueue)
631 };
632 let early = budget.allow_early_flush(now, queued, last_enq);
633 let max_linger = self.shared.flush_config.linger;
634 let eff_linger = if !budget.config.enabled {
635 max_linger
636 } else if early || max_linger.is_zero() {
637 Duration::ZERO
638 } else {
639 max_linger
640 };
641 PipelineSnapshot {
642 budget_per_sec: EffectiveKnob {
643 configured: budget.config.budget_per_sec_cap,
644 startup_measured: budget.startup_capacity,
645 ongoing_measured: budget.ongoing_capacity,
646 effective,
647 reason,
648 },
649 effective_linger_ms: EffectiveKnob {
650 configured: Some(max_linger.as_millis() as u64),
651 startup_measured: None,
652 ongoing_measured: None,
653 effective: eff_linger.as_millis() as u64,
654 reason: if early {
655 EffectiveReason::Ongoing
656 } else {
657 EffectiveReason::Configured
658 },
659 },
660 max_linger_ms: max_linger.as_millis() as u64,
661 token_fill_ratio: budget.fill_ratio(),
662 media_ops_total: budget.media_ops_total,
663 overdraft_total: budget.overdraft_total,
664 flushes_total: budget.flushes_total,
665 undersized_deadline_flushes: budget.undersized_deadline_flushes,
666 predicted_media_ops: budget.predicted_media_ops,
667 budget_enabled: budget.config.enabled,
668 budget_mode: budget.config.mode,
669 }
670 }
671}
672
673pub async fn reap_orphans(
679 blob: &dyn BlobStore,
680 data_prefix: &str,
681 live: &HashSet<String>,
682) -> Result<Vec<String>, ObjectLogError> {
683 let keys = blob.list(data_prefix).await?;
684 let mut deleted = Vec::new();
685 for key in keys {
686 if live.contains(&key) {
687 continue;
688 }
689 blob.delete(&key).await?;
690 deleted.push(key);
691 }
692 Ok(deleted)
693}
694
695impl<S: Sequencer> Drop for LogEngine<S> {
696 fn drop(&mut self) {
697 {
698 let mut q = self.shared.queue.lock().expect("poisoned");
699 q.shutdown = true;
700 }
701 self.shared.cv.notify_all();
702 if let Some(t) = self.flush_thread.take() {
703 let _ = t.join();
704 }
705 }
706}
707
708async fn recover_data_object_counter(
714 blob: &dyn BlobStore,
715 prefix: &str,
716 writer_id: &str,
717) -> Result<u64, ObjectLogError> {
718 let keys = blob.list(prefix).await?;
719 let mut max = 0u64;
720 let writer_prefix = format!("{prefix}{writer_id}/");
721 for key in keys {
722 let Some(suffix) = key.strip_prefix(prefix) else {
723 continue;
724 };
725 if writer_id.is_empty() {
726 if let Ok(n) = suffix.parse::<u64>() {
727 max = max.max(n);
728 }
729 continue;
730 }
731 let Some(rest) = key.strip_prefix(&writer_prefix) else {
732 continue;
733 };
734 if let Some(n) = rest
735 .rsplit('/')
736 .next()
737 .and_then(|tail| tail.parse::<u64>().ok())
738 {
739 max = max.max(n);
740 }
741 }
742 Ok(max)
743}
744
745fn data_object_key(prefix: &str, writer_id: &str, epoch: u64, counter: u64) -> String {
746 if writer_id.is_empty() {
747 format!("{prefix}{counter:020}")
748 } else {
749 format!("{prefix}{writer_id}/{epoch:020}/{counter:020}")
750 }
751}
752
753fn flush_loop<S>(
754 shared: Arc<Shared<S::Meta>>,
755 blob: Arc<dyn BlobStore>,
756 sequencer: Arc<S>,
757 config: FlushConfig,
758 prefix: String,
759 writer_id: String,
760) where
761 S: Sequencer + 'static,
762 S::Meta: Send + 'static,
763{
764 let max_inflight = config.max_inflight_flushes.max(1);
765 let commit_group_limit = if sequencer.supports_multi_object_commit() {
766 max_inflight
767 } else {
768 1
769 };
770 let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
771 .ok()
772 .and_then(|v| v.parse::<usize>().ok())
773 .filter(|v| *v > 0)
774 .unwrap_or_else(|| max_inflight.min(8));
775 let rt = tokio::runtime::Builder::new_multi_thread()
776 .worker_threads(worker_threads)
777 .enable_all()
778 .build()
779 .expect("flush runtime");
780 let mut counter = match rt.block_on(recover_data_object_counter(
784 blob.as_ref(),
785 &prefix,
786 &writer_id,
787 )) {
788 Ok(counter) => counter,
789 Err(error) => {
790 let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
793 return;
794 }
795 };
796 let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
797 let mut active_puts = 0usize;
798 let mut committing: Option<CommitJob> = None;
799 let mut shutdown = false;
800
801 loop {
802 while !shutdown && active_puts < max_inflight {
803 let wait_for_more = pending.is_empty() && committing.is_none();
806 match take_batch(&shared, config, wait_for_more) {
807 TakeBatch::Batch(batch) => {
808 counter += 1;
809 let concurrent = max_inflight > 1;
810 let work = start_flush_work(
811 &rt, &blob, &prefix, &writer_id, counter, batch, concurrent,
812 );
813 if concurrent {
814 active_puts += 1;
815 }
816 pending.push_back(work);
817 }
818 TakeBatch::Empty => break,
819 TakeBatch::Shutdown => shutdown = true,
820 }
821 }
822
823 let mut made_progress = false;
824 for work in pending.iter_mut() {
825 let Some(put) = work.put.as_ref() else {
826 continue;
827 };
828 if !put.is_finished() {
829 continue;
830 }
831 let put = work.put.take().expect("put handle exists");
832 let elapsed = work.put_started.elapsed();
833 work.put_result = Some(match rt.block_on(put) {
834 Ok(Ok(())) => Ok(elapsed),
835 Ok(Err(e)) => Err(e),
836 Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
837 "flush task failed: {e}"
838 ))),
839 });
840 active_puts = active_puts.saturating_sub(1);
841 made_progress = true;
842 }
843
844 if committing
845 .as_ref()
846 .is_some_and(|job| job.task.is_finished())
847 {
848 let job = committing.take().expect("completed commit exists");
849 match rt.block_on(job.task) {
850 Ok(released) => {
851 let mut q = shared.queue.lock().expect("poisoned");
852 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
853 shared.cv.notify_all();
854 }
855 Err(error) => {
856 let uploads = fail_engine_and_take_pending_uploads(
857 &shared,
858 &mut pending,
859 job.first_seq,
860 ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
861 );
862 for upload in uploads {
866 let _ = rt.block_on(upload);
867 }
868 return;
869 }
870 }
871 made_progress = true;
872 }
873
874 if committing.is_none() {
875 if let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit) {
876 if max_inflight == 1 {
877 let released = finish_flush_work(&shared, &blob, &sequencer, work);
879 let mut q = shared.queue.lock().expect("poisoned");
880 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
881 shared.cv.notify_all();
882 } else {
883 let first_seq = work
886 .batch
887 .iter()
888 .map(|p| p.seq)
889 .min()
890 .unwrap_or(work.max_seq);
891 let shared = shared.clone();
892 let blob = blob.clone();
893 let sequencer = sequencer.clone();
894 committing = Some(CommitJob {
895 task: rt.spawn_blocking(move || {
896 finish_flush_work(&shared, &blob, &sequencer, work)
897 }),
898 first_seq,
899 });
900 }
901 made_progress = true;
902 }
903 }
904
905 if pending.is_empty() && committing.is_none() {
906 {
907 let mut q = shared.queue.lock().expect("poisoned");
908 if q.items.is_empty() && q.bytes_in_use == 0 {
909 notify_flush_waiters(&mut q);
912 }
913 }
914 shared.cv.notify_all();
915 if shutdown {
916 let mut q = shared.queue.lock().expect("poisoned");
918 for (_, tx) in q.flush_waiters.drain(..) {
919 let _ = tx.send(Err(ObjectLogError::Sequencer(
920 "engine shutting down".into(),
921 )));
922 }
923 return;
924 }
925 continue;
926 }
927
928 if !made_progress {
929 std::thread::sleep(Duration::from_millis(1));
930 }
931 }
932}
933
934fn fail_engine_and_take_pending_uploads<M>(
937 shared: &Arc<Shared<M>>,
938 pending: &mut VecDeque<FlushWork<M>>,
939 first_seq: u64,
940 error: ObjectLogError,
941) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
942 let mut uploads = Vec::new();
943 for mut work in pending.drain(..) {
944 if let Some(put) = work.put.take() {
945 uploads.push(put);
946 }
947 send_storage_error(&mut work.responders, error.clone());
948 }
949 let mut q = shared
951 .queue
952 .lock()
953 .unwrap_or_else(|poisoned| poisoned.into_inner());
954 q.shutdown = true;
955 q.first_failed.get_or_insert((first_seq, error.clone()));
956 for mut item in q.items.drain(..) {
957 if let Some(tx) = item.responder.take() {
958 let _ = tx.send(Err(error.clone()));
959 }
960 }
961 q.bytes = 0;
962 for (_, tx) in q.flush_waiters.drain(..) {
965 let _ = tx.send(Err(error.clone()));
966 }
967 q.force_flush = false;
968 shared.cv.notify_all();
969 uploads
970}
971
972fn take_ready_commit_group<M>(
975 pending: &mut VecDeque<FlushWork<M>>,
976 max_objects: usize,
977) -> Option<FlushWork<M>> {
978 pending.front()?.put_result.as_ref()?;
979 let mut group = pending.pop_front().expect("ready front exists");
980 if group.put_result.as_ref().is_some_and(Result::is_err) {
981 return Some(group);
982 }
983 while group.data_objects < max_objects.max(1) {
984 let Some(next) = pending.front() else { break };
985 if !next.put_result.as_ref().is_some_and(Result::is_ok)
986 || group.data_objects.saturating_add(next.data_objects) > max_objects
987 {
988 break;
989 }
990 let mut next = pending.pop_front().expect("ready successor exists");
991 group.batch.append(&mut next.batch);
992 group.locations.append(&mut next.locations);
993 group.responders.append(&mut next.responders);
994 group.bytes = group
995 .bytes
996 .checked_add(next.bytes)
997 .expect("bounded pending bytes");
998 group.max_seq = group.max_seq.max(next.max_seq);
999 group.data_objects += next.data_objects;
1000 let next_elapsed = next.put_result.take().unwrap().unwrap();
1001 let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
1002 *elapsed = (*elapsed).max(next_elapsed);
1003 }
1004 Some(group)
1005}
1006
1007fn effective_linger<M>(
1011 shared: &Shared<M>,
1012 config: FlushConfig,
1013 queued_bytes: usize,
1014 last_enqueue: Option<Instant>,
1015 oldest_enqueue: Option<Instant>,
1016) -> Duration {
1017 if config.linger.is_zero() {
1018 return Duration::ZERO;
1019 }
1020 let now = Instant::now();
1021 if config.budget.enabled {
1022 let mut budget = shared.budget.lock().expect("poisoned");
1023 budget.refill(now);
1024 if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
1025 budget.note_early_flush(now);
1026 return Duration::ZERO;
1027 }
1028 }
1029 if let Some(oldest) = oldest_enqueue {
1031 let deadline = oldest + config.linger;
1032 if now >= deadline {
1033 return Duration::ZERO;
1034 }
1035 let until_deadline = deadline.saturating_duration_since(now);
1036 if config.budget.enabled
1038 && !config.budget.early_flush_idle.is_zero()
1039 && let Some(last) = last_enqueue
1040 {
1041 let idle_at = last + config.budget.early_flush_idle;
1042 if idle_at > now && idle_at < deadline {
1043 return idle_at.saturating_duration_since(now);
1044 }
1045 }
1046 return until_deadline;
1047 }
1048 config.linger
1049}
1050
1051fn take_batch<M>(
1052 shared: &Arc<Shared<M>>,
1053 config: FlushConfig,
1054 wait_when_empty: bool,
1055) -> TakeBatch<M> {
1056 let mut q = shared.queue.lock().expect("poisoned");
1057 loop {
1058 if q.items.is_empty() {
1059 if q.shutdown {
1060 return TakeBatch::Shutdown;
1061 }
1062 if !wait_when_empty {
1063 return TakeBatch::Empty;
1064 }
1065 q = shared.cv.wait(q).expect("poisoned");
1069 continue;
1070 }
1071
1072 let linger = {
1073 let queued = q.bytes;
1074 let last_enq = q.last_enqueue;
1075 let oldest = q.oldest_enqueue;
1076 drop(q);
1077 let l = effective_linger(shared, config, queued, last_enq, oldest);
1078 q = shared.queue.lock().expect("poisoned");
1079 l
1080 };
1081
1082 let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1083 let force = q.force_flush;
1084 let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1085 if triggered {
1086 if !size_trigger && !q.shutdown && !q.items.is_empty() {
1087 let mut budget = shared.budget.lock().expect("poisoned");
1088 if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1089 budget.undersized_deadline_flushes += 1;
1090 }
1091 }
1092 break;
1093 }
1094
1095 let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1098 q = guard;
1099 }
1100
1101 let force_drain = q.force_flush;
1102 let mut items = Vec::new();
1103 let mut bytes = 0usize;
1104 while let Some(item) = q.items.pop_front() {
1107 q.bytes = q.bytes.saturating_sub(item.payload.len());
1108 bytes += item.payload.len();
1109 items.push(item);
1110 if bytes >= config.max_bytes {
1111 break;
1112 }
1113 if !force_drain && items.len() >= config.max_batches {
1114 break;
1115 }
1116 }
1117 q.oldest_enqueue = if q.items.is_empty() {
1118 None
1119 } else {
1120 Some(Instant::now())
1122 };
1123 if force_drain {
1125 q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1126 }
1127 if items.is_empty() {
1128 TakeBatch::Empty
1129 } else {
1130 TakeBatch::Batch(items)
1131 }
1132}
1133
1134fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1135 let done = q.completed_through;
1136 let mut i = 0;
1137 while i < q.flush_waiters.len() {
1138 let barrier = q.flush_waiters[i].0;
1139 let failure = q
1140 .first_failed
1141 .as_ref()
1142 .filter(|(sequence, _)| *sequence <= barrier)
1143 .map(|(_, error)| error.clone());
1144 if barrier <= done || failure.is_some() {
1145 let (_, tx) = q.flush_waiters.swap_remove(i);
1146 let _ = tx.send(failure.map_or(Ok(()), Err));
1147 } else {
1148 i += 1;
1149 }
1150 }
1151 if q.flush_waiters.is_empty() {
1152 q.force_flush = false;
1153 } else if q.items.is_empty() && q.bytes_in_use == 0 {
1154 q.force_flush = true;
1156 } else {
1157 q.force_flush = true;
1158 }
1159}
1160
1161fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1162 for (_, tx) in responders.iter_mut() {
1163 if let Some(tx) = tx.take() {
1164 let _ = tx.send(Err(err.clone()));
1165 }
1166 }
1167}
1168
1169fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1170 for (durability, tx) in responders.iter_mut() {
1171 if *durability == Durability::Durable
1172 && let Some(tx) = tx.take()
1173 {
1174 let _ = tx.send(Ok(AppendOutcome {
1175 base_offset: None,
1176 last_offset: None,
1177 durable: true,
1178 sequenced: false,
1179 }));
1180 }
1181 }
1182}
1183
1184fn prepare_flush_work<M>(
1185 prefix: &str,
1186 writer_id: &str,
1187 counter: u64,
1188 mut batch: Vec<Pending<M>>,
1189) -> (FlushWork<M>, String, Vec<Bytes>) {
1190 let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1191 let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1192 let epoch = batch.first().map(|pending| pending.epoch).unwrap_or(0);
1193 let key = data_object_key(prefix, writer_id, epoch, counter);
1194 let mut offset = 0usize;
1195 for p in &batch {
1196 let start = offset as u32;
1197 offset += p.payload.len();
1198 chunks.push(p.payload.clone());
1199 locations.push(BatchLocation {
1200 object_id: key.clone(),
1201 byte_start: start,
1202 byte_len: p.payload.len() as u32,
1203 });
1204 }
1205 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1206 eprintln!(
1207 "object-log seal: key={key} batches={} bytes={offset}",
1208 batch.len()
1209 );
1210 }
1211 let responders: Vec<(Durability, Option<Responder>)> = batch
1212 .iter_mut()
1213 .map(|p| (p.durability, p.responder.take()))
1214 .collect();
1215 let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1216 let work = FlushWork {
1217 batch,
1218 locations,
1219 responders,
1220 bytes: offset,
1221 max_seq,
1222 data_objects: 1,
1223 put: None,
1224 put_started: Instant::now(),
1225 put_result: None,
1226 };
1227 (work, key, chunks)
1228}
1229
1230fn start_flush_work<M>(
1236 rt: &tokio::runtime::Runtime,
1237 blob: &Arc<dyn BlobStore>,
1238 prefix: &str,
1239 writer_id: &str,
1240 counter: u64,
1241 batch: Vec<Pending<M>>,
1242 concurrent: bool,
1243) -> FlushWork<M> {
1244 let (mut work, key, chunks) = prepare_flush_work(prefix, writer_id, counter, batch);
1245 let blob = Arc::clone(blob);
1246 let _ = blob.take_media_op_stats();
1247 work.put_started = Instant::now();
1248 if concurrent {
1249 work.put =
1250 Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
1251 } else {
1252 let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1253 let elapsed = work.put_started.elapsed();
1254 work.put_result = Some(match put_result {
1255 Ok(()) => Ok(elapsed),
1256 Err(e) => Err(e),
1257 });
1258 }
1259 work
1260}
1261
1262fn finish_flush_work<S, M>(
1263 shared: &Arc<Shared<M>>,
1264 blob: &Arc<dyn BlobStore>,
1265 sequencer: &Arc<S>,
1266 mut work: FlushWork<S::Meta>,
1267) -> usize
1268where
1269 S: Sequencer + 'static,
1270 S::Meta: Send + 'static,
1271{
1272 let release_bytes = work.bytes;
1273 let first_seq = work
1274 .batch
1275 .iter()
1276 .map(|pending| pending.seq)
1277 .min()
1278 .unwrap_or(work.max_seq);
1279 let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1283 let put_elapsed = match work.put_result.take().expect("put result is ready") {
1284 Ok(elapsed) => elapsed,
1285 Err(e) => {
1286 send_storage_error(&mut work.responders, e.clone());
1287 let mut q = shared.queue.lock().expect("poisoned");
1288 let done = work.max_seq;
1290 if done >= q.completed_through {
1291 q.completed_through = done;
1292 }
1293 q.first_failed.get_or_insert((first_seq, e));
1294 notify_flush_waiters(&mut q);
1295 shared.cv.notify_all();
1296 return release_bytes;
1297 }
1298 };
1299
1300 let mut media_ops = blob
1302 .take_media_op_stats()
1303 .map(|s| s.media_ops)
1304 .unwrap_or(work.data_objects as u64); send_durable_acks(&mut work.responders);
1308
1309 let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1311 .batch
1312 .iter()
1313 .zip(work.locations.iter())
1314 .map(|(p, loc)| CommitBatch {
1315 partition: p.partition.clone(),
1316 record_count: p.record_count,
1317 location: loc.clone(),
1318 meta: &p.meta,
1319 epoch: p.epoch,
1320 })
1321 .collect();
1322
1323 let commit_started = timing.then(Instant::now);
1324 let _ = blob.take_media_op_stats();
1327 let mut commit_error = None;
1328 match sequencer.commit(&commit_batches) {
1329 Ok(outcomes) => {
1330 if let Some(commit_started) = commit_started {
1331 eprintln!(
1332 "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1333 work.bytes,
1334 commit_batches.len(),
1335 put_elapsed.as_millis(),
1336 commit_started.elapsed().as_millis(),
1337 work.data_objects
1338 );
1339 }
1340 if let Some(s) = blob.take_media_op_stats() {
1341 media_ops = media_ops.saturating_add(s.media_ops);
1342 }
1343 for ((outcome, (_, tx)), pending) in outcomes
1344 .into_iter()
1345 .zip(work.responders.iter_mut())
1346 .zip(work.batch.iter())
1347 {
1348 if let Some(tx) = tx.take() {
1349 let (base, last) = match outcome {
1350 CommitOutcome::Assigned {
1351 base_offset,
1352 record_count,
1353 } => (
1354 Some(base_offset),
1355 Some(base_offset + record_count as i64 - 1),
1356 ),
1357 CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1358 CommitOutcome::Rejected { reason } => {
1359 let error = sequencer.rejection_error(
1361 &pending.partition,
1362 pending.epoch,
1363 reason,
1364 );
1365 let _ = tx.send(Err(error));
1366 continue;
1367 }
1368 };
1369 let _ = tx.send(Ok(AppendOutcome {
1370 base_offset: base,
1371 last_offset: last,
1372 durable: true,
1373 sequenced: true,
1374 }));
1375 }
1376 }
1377 }
1378 Err(e) => {
1379 if let Some(s) = blob.take_media_op_stats() {
1380 media_ops = media_ops.saturating_add(s.media_ops);
1381 }
1382 send_storage_error(&mut work.responders, e.clone());
1383 commit_error = Some(e);
1384 }
1385 }
1386
1387 {
1388 let mut budget = shared.budget.lock().expect("poisoned");
1389 budget.consume_after_flush(media_ops, Instant::now());
1390 }
1391 {
1392 let mut q = shared.queue.lock().expect("poisoned");
1393 if let Some(error) = commit_error {
1394 q.first_failed.get_or_insert((first_seq, error));
1395 }
1396 if work.max_seq >= q.completed_through {
1397 q.completed_through = work.max_seq;
1398 }
1399 notify_flush_waiters(&mut q);
1400 }
1401 shared.cv.notify_all();
1402
1403 release_bytes
1404}
1405
1406#[cfg(test)]
1407mod flush_barrier_tests {
1408 use super::*;
1409
1410 #[test]
1411 fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1412 let error = ObjectLogError::Sequencer("manifest failed".into());
1413 let mut queue = Queue::<()> {
1414 items: VecDeque::new(),
1415 bytes: 0,
1416 bytes_in_use: 1,
1417 shutdown: false,
1418 next_seq: 4,
1419 completed_through: 2,
1420 first_failed: Some((2, error.clone())),
1421 force_flush: true,
1422 flush_waiters: Vec::new(),
1423 last_enqueue: None,
1424 oldest_enqueue: None,
1425 };
1426 let mut receivers = Vec::new();
1427 for barrier in [1, 2, 3] {
1428 let (tx, rx) = oneshot::channel();
1429 queue.flush_waiters.push((barrier, tx));
1430 receivers.push(rx);
1431 }
1432 notify_flush_waiters(&mut queue);
1433 assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1434 assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1435 assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1436 assert!(queue.flush_waiters.is_empty());
1437 }
1438}
1439
1440#[cfg(test)]
1441mod ready_commit_group_tests {
1442 use super::*;
1443 use crate::{ManifestSequencer, MemoryBlobStore};
1444 use async_trait::async_trait;
1445 use std::ops::Range;
1446
1447 fn work(
1448 n: u64,
1449 durability: Durability,
1450 ) -> (
1451 FlushWork<()>,
1452 String,
1453 Vec<Bytes>,
1454 oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1455 ) {
1456 let (tx, rx) = oneshot::channel();
1457 let (mut work, key, chunks) = prepare_flush_work(
1458 "data/",
1459 "",
1460 n,
1461 vec![Pending {
1462 partition: PartitionKey("shared".into()),
1463 record_count: 1,
1464 payload: Bytes::from(format!("body-{n}")),
1465 meta: (),
1466 epoch: 0,
1467 durability,
1468 responder: Some(tx),
1469 seq: n,
1470 }],
1471 );
1472 work.put_result = Some(Ok(Duration::from_millis(n)));
1473 (work, key, chunks, rx)
1474 }
1475
1476 #[test]
1477 fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1478 let mut pending = (1..=5)
1479 .map(|n| work(n, Durability::Sequenced).0)
1480 .collect::<VecDeque<_>>();
1481 let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1482 let mut released = 0;
1483 for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1484 let group = take_ready_commit_group(&mut pending, 2).unwrap();
1485 assert_eq!(
1486 group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1487 expected
1488 );
1489 assert_eq!(group.data_objects, expected.len());
1490 assert_eq!(group.responders.len(), expected.len());
1491 assert_eq!(group.max_seq, *expected.last().unwrap());
1492 for (location, n) in group.locations.iter().zip(&expected) {
1493 assert_eq!(location.object_id, format!("data/{n:020}"));
1494 assert_eq!(location.byte_start, 0);
1495 assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1496 }
1497 released += group.bytes;
1498 }
1499 assert!(pending.is_empty());
1500 assert_eq!(released, original_bytes);
1501
1502 let mut first = work(1, Durability::Sequenced).0;
1503 first.put_result = None;
1504 let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1505 assert!(take_ready_commit_group(&mut pending, 8).is_none());
1506 assert_eq!(
1507 pending.len(),
1508 2,
1509 "a later ready PUT cannot pass an unfinished head"
1510 );
1511 pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1512 let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1513 assert_eq!(failed.data_objects, 1);
1514 assert!(failed.put_result.unwrap().is_err());
1515 assert_eq!(pending[0].max_seq, 2);
1516
1517 let mut failed = work(2, Durability::Sequenced).0;
1518 failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1519 let mut pending = VecDeque::from([
1520 work(1, Durability::Sequenced).0,
1521 failed,
1522 work(3, Durability::Sequenced).0,
1523 ]);
1524 assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1525 assert_eq!(
1526 pending.len(),
1527 2,
1528 "a success group cannot cross a failed PUT"
1529 );
1530 let mut pending = VecDeque::from([
1531 work(1, Durability::Sequenced).0,
1532 work(2, Durability::Sequenced).0,
1533 ]);
1534 assert_eq!(
1535 take_ready_commit_group(&mut pending, 1)
1536 .unwrap()
1537 .data_objects,
1538 1
1539 );
1540 assert_eq!(
1541 pending.len(),
1542 1,
1543 "one-object sequencers retain their call boundary"
1544 );
1545 }
1546
1547 struct GatedManifestStore {
1548 inner: MemoryBlobStore,
1549 entered: std::sync::mpsc::SyncSender<()>,
1550 release: Mutex<Option<oneshot::Receiver<()>>>,
1551 fail: bool,
1552 panic_on_commit: bool,
1553 }
1554
1555 #[async_trait]
1556 impl BlobStore for GatedManifestStore {
1557 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1558 if key.starts_with("manifest/") {
1559 let release = self.release.lock().unwrap().take();
1560 if let Some(release) = release {
1561 self.entered.send(()).unwrap();
1562 release.await.unwrap();
1563 assert!(!self.panic_on_commit, "injected commit panic");
1564 if self.fail {
1565 return Err(ObjectLogError::StorageUnavailable(
1566 "manifest failure".into(),
1567 ));
1568 }
1569 }
1570 }
1571 self.inner.put(key, value).await
1572 }
1573 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1574 self.inner.get(key).await
1575 }
1576 async fn get_range(
1577 &self,
1578 key: &str,
1579 range: Range<u64>,
1580 ) -> Result<Option<Bytes>, ObjectLogError> {
1581 self.inner.get_range(key, range).await
1582 }
1583 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1584 self.inner.list(prefix).await
1585 }
1586 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1587 self.inner.delete(key).await
1588 }
1589 fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1590 self.inner.take_media_op_stats()
1591 }
1592 }
1593
1594 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1595 async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1596 for fail in [false, true] {
1597 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1598 let (release_tx, release_rx) = oneshot::channel();
1599 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1600 inner: MemoryBlobStore::new(),
1601 entered: entered_tx,
1602 release: Mutex::new(Some(release_rx)),
1603 fail,
1604 panic_on_commit: false,
1605 });
1606 let sequencer = Arc::new(
1607 ManifestSequencer::open(blob.clone(), "manifest/")
1608 .await
1609 .unwrap(),
1610 );
1611 assert!(sequencer.supports_multi_object_commit());
1612 let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1613 let (mut second, second_key, second_chunks, mut durable_rx) =
1614 work(2, Durability::Durable);
1615 second.batch[0].partition = PartitionKey("other".into());
1616 let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1617 blob.put_chunks(&first_key, first_chunks).await.unwrap();
1618 blob.put_chunks(&second_key, second_chunks).await.unwrap();
1619 blob.put_chunks(&third_key, third_chunks).await.unwrap();
1620 let mut pending = VecDeque::from([first, second, third]);
1621 let group = take_ready_commit_group(&mut pending, 8).unwrap();
1622 let bytes = group.bytes;
1623 assert_eq!(group.data_objects, 3);
1624 let (flush_tx, mut flush_rx) = oneshot::channel();
1625 let mut config = FlushConfig::default();
1626 config.budget.enabled = false;
1627 let shared: Arc<Shared<()>> = Arc::new(Shared {
1628 queue: Mutex::new(Queue {
1629 items: VecDeque::new(),
1630 bytes: 0,
1631 bytes_in_use: bytes,
1632 shutdown: false,
1633 next_seq: 4,
1634 completed_through: 0,
1635 first_failed: None,
1636 force_flush: true,
1637 flush_waiters: vec![(3, flush_tx)],
1638 last_enqueue: None,
1639 oldest_enqueue: None,
1640 }),
1641 cv: Condvar::new(),
1642 max_buffered_bytes: bytes,
1643 budget: Mutex::new(BudgetRuntime::new(config.budget)),
1644 flush_config: config,
1645 });
1646 let worker = {
1647 let shared = shared.clone();
1648 let blob = blob.clone();
1649 let sequencer = sequencer.clone();
1650 std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1651 };
1652 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1653 assert!(matches!(
1654 sequenced_rx.try_recv(),
1655 Err(oneshot::error::TryRecvError::Empty)
1656 ));
1657 assert!(matches!(
1658 third_rx.try_recv(),
1659 Err(oneshot::error::TryRecvError::Empty)
1660 ));
1661 assert!(matches!(
1662 flush_rx.try_recv(),
1663 Err(oneshot::error::TryRecvError::Empty)
1664 ));
1665 let durable = durable_rx.try_recv().unwrap().unwrap();
1666 assert!(durable.durable && !durable.sequenced);
1667 release_tx.send(()).unwrap();
1668 assert_eq!(worker.join().unwrap(), bytes);
1669 if fail {
1670 assert!(sequenced_rx.await.unwrap().is_err());
1671 assert!(third_rx.await.unwrap().is_err());
1672 assert!(flush_rx.await.unwrap().is_err());
1673 assert_eq!(
1674 sequencer
1675 .high_watermark(&PartitionKey("shared".into()))
1676 .unwrap(),
1677 0
1678 );
1679 assert!(blob.list("manifest/").await.unwrap().is_empty());
1680 } else {
1681 assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1682 assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1683 flush_rx.await.unwrap().unwrap();
1684 assert_eq!(blob.list("manifest/").await.unwrap().len(), 3);
1685 }
1686 assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1687 let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1688 .await
1689 .unwrap();
1690 for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1691 let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1692 assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1693 for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1694 assert_eq!(entry.base_offset, offset as i64);
1695 let loc = &entry.location;
1696 assert_eq!(loc.object_id, format!("data/{object:020}"));
1697 let body = blob
1698 .get_range(
1699 &loc.object_id,
1700 loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1701 )
1702 .await
1703 .unwrap()
1704 .unwrap();
1705 assert_eq!(body, format!("body-{object}"));
1706 }
1707 }
1708 }
1709 }
1710
1711 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1712 async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1713 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1714 let (release_tx, release_rx) = oneshot::channel();
1715 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1716 inner: MemoryBlobStore::new(),
1717 entered: entered_tx,
1718 release: Mutex::new(Some(release_rx)),
1719 fail: false,
1720 panic_on_commit: false,
1721 });
1722 let sequencer = Arc::new(
1723 ManifestSequencer::open(blob.clone(), "manifest/")
1724 .await
1725 .unwrap(),
1726 );
1727 let mut config = FlushConfig::default();
1728 config.max_batches = 1;
1729 config.max_inflight_flushes = 4;
1730 config.linger = Duration::ZERO;
1731 config.budget.enabled = false;
1732 let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1733 let first = {
1734 let engine = engine.clone();
1735 tokio::spawn(async move {
1736 engine
1737 .produce(
1738 PartitionKey("p".into()),
1739 Bytes::from_static(b"first"),
1740 1,
1741 (),
1742 Durability::Sequenced,
1743 )
1744 .await
1745 })
1746 };
1747 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1748 for n in 1..4 {
1749 engine
1750 .produce(
1751 PartitionKey("p".into()),
1752 Bytes::from(format!("later-{n}")),
1753 1,
1754 (),
1755 Durability::Buffered,
1756 )
1757 .await
1758 .unwrap();
1759 }
1760 let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1761 loop {
1762 if blob.list("data/").await.unwrap().len() == 4 {
1763 break;
1764 }
1765 tokio::time::sleep(Duration::from_millis(1)).await;
1766 }
1767 })
1768 .await
1769 .is_ok();
1770 let early_ack = first.is_finished();
1771 release_tx.send(()).unwrap();
1773 first.await.unwrap().unwrap();
1774 engine.flush().await.unwrap();
1775 let rows = engine
1776 .fetch(&PartitionKey("p".into()), 0, 1024)
1777 .await
1778 .unwrap();
1779 assert_eq!(
1780 rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
1781 vec![0, 1, 2, 3]
1782 );
1783 drop(engine);
1784 assert!(
1785 !early_ack,
1786 "sequenced acknowledgement preceded manifest durability"
1787 );
1788 assert!(
1789 upload_progress,
1790 "blocked manifest prevented independent data uploads"
1791 );
1792 }
1793
1794 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1795 async fn commit_completion_releases_admission_bytes_without_more_uploads() {
1796 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1797 let sequencer = Arc::new(
1798 ManifestSequencer::open(blob.clone(), "manifest/")
1799 .await
1800 .unwrap(),
1801 );
1802 let mut config = FlushConfig::default();
1803 config.max_bytes = 4;
1804 config.max_buffered_bytes = 4;
1805 config.max_batches = 1;
1806 config.max_inflight_flushes = 4;
1807 config.linger = Duration::ZERO;
1808 config.budget.enabled = false;
1809 let engine = LogEngine::new(blob, sequencer, config, "data/");
1810 engine
1811 .produce(
1812 PartitionKey("p".into()),
1813 Bytes::from_static(b"full"),
1814 1,
1815 (),
1816 Durability::Sequenced,
1817 )
1818 .await
1819 .unwrap();
1820 tokio::time::timeout(Duration::from_secs(5), async {
1821 while engine.buffer_stats().bytes_in_use != 0 {
1822 tokio::time::sleep(Duration::from_millis(1)).await;
1823 }
1824 })
1825 .await
1826 .expect("completed committer stranded admission bytes");
1827 engine
1828 .produce(
1829 PartitionKey("p".into()),
1830 Bytes::from_static(b"next"),
1831 1,
1832 (),
1833 Durability::Sequenced,
1834 )
1835 .await
1836 .unwrap();
1837 engine.flush().await.unwrap();
1838 assert_eq!(
1839 engine
1840 .fetch(&PartitionKey("p".into()), 0, 100)
1841 .await
1842 .unwrap()
1843 .len(),
1844 2
1845 );
1846 }
1847
1848 struct FailureInjectingStore {
1849 inner: Arc<dyn BlobStore>,
1850 second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
1851 second_upload_completed: std::sync::atomic::AtomicBool,
1852 fail_data_listing: std::sync::atomic::AtomicBool,
1853 }
1854
1855 impl FailureInjectingStore {
1856 fn new(inner: Arc<dyn BlobStore>) -> Self {
1857 Self {
1858 inner,
1859 second_upload_gate: Mutex::new(None),
1860 second_upload_completed: std::sync::atomic::AtomicBool::new(false),
1861 fail_data_listing: std::sync::atomic::AtomicBool::new(false),
1862 }
1863 }
1864 }
1865
1866 #[async_trait]
1867 impl BlobStore for FailureInjectingStore {
1868 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1869 let gate = if key == "data/00000000000000000002" {
1870 self.second_upload_gate.lock().unwrap().take()
1871 } else {
1872 None
1873 };
1874 if let Some((entered, release)) = gate {
1875 entered.send(()).unwrap();
1876 release.await.unwrap();
1877 let result = self.inner.put(key, value).await;
1878 if result.is_ok() {
1879 self.second_upload_completed
1880 .store(true, std::sync::atomic::Ordering::SeqCst);
1881 }
1882 return result;
1883 }
1884 self.inner.put(key, value).await
1885 }
1886 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1887 self.inner.get(key).await
1888 }
1889 async fn get_range(
1890 &self,
1891 key: &str,
1892 range: Range<u64>,
1893 ) -> Result<Option<Bytes>, ObjectLogError> {
1894 self.inner.get_range(key, range).await
1895 }
1896 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1897 if prefix == "data/"
1898 && self
1899 .fail_data_listing
1900 .load(std::sync::atomic::Ordering::SeqCst)
1901 {
1902 return Err(ObjectLogError::StorageUnavailable(
1903 "injected object listing failure".into(),
1904 ));
1905 }
1906 self.inner.list(prefix).await
1907 }
1908 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1909 self.inner.delete(key).await
1910 }
1911 }
1912
1913 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1914 async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
1915 let inner = Arc::new(MemoryBlobStore::new());
1916 let key = "data/00000000000000000001";
1917 let original = Bytes::from_static(b"existing immutable object");
1918 inner.put(key, original.clone()).await.unwrap();
1919 let store = Arc::new(FailureInjectingStore::new(inner.clone()));
1920 store
1921 .fail_data_listing
1922 .store(true, std::sync::atomic::Ordering::SeqCst);
1923 let mut config = FlushConfig::default();
1924 config.max_batches = 1;
1925 config.max_inflight_flushes = 4;
1926 config.linger = Duration::ZERO;
1927 config.budget.enabled = false;
1928 let engine = LogEngine::new(
1929 store,
1930 Arc::new(crate::InMemorySequencer::new()),
1931 config,
1932 "data/",
1933 );
1934 let result = tokio::time::timeout(
1935 Duration::from_secs(5),
1936 engine.produce(
1937 PartitionKey("p".into()),
1938 Bytes::from_static(b"must not overwrite"),
1939 1,
1940 (),
1941 Durability::Sequenced,
1942 ),
1943 )
1944 .await;
1945 let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
1946 tokio::task::spawn_blocking(move || drop(engine))
1947 .await
1948 .unwrap();
1949 let retained = inner.get(key).await.unwrap();
1950 assert_eq!(
1951 retained,
1952 Some(original),
1953 "failed listing must preserve sealed bytes"
1954 );
1955 assert!(
1956 result.unwrap().is_err(),
1957 "failed prefix recovery must reject admission"
1958 );
1959 assert!(
1960 flushed.unwrap().is_err(),
1961 "flush must report failed initialization"
1962 );
1963 assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
1964 }
1965
1966 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1967 async fn engines_commit_and_close_independently() {
1968 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1969 let (release_tx, release_rx) = oneshot::channel();
1970 let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1971 inner: MemoryBlobStore::new(),
1972 entered: entered_tx,
1973 release: Mutex::new(Some(release_rx)),
1974 fail: false,
1975 panic_on_commit: false,
1976 })));
1977 let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
1978 let mut config = FlushConfig::default();
1979 config.max_batches = 1;
1980 config.max_inflight_flushes = 4;
1981 config.linger = Duration::ZERO;
1982 config.budget.enabled = false;
1983 let a_seq = Arc::new(
1984 ManifestSequencer::open(a_blob.clone(), "manifest/")
1985 .await
1986 .unwrap(),
1987 );
1988 let b_seq = Arc::new(
1989 ManifestSequencer::open(b_blob.clone(), "manifest/")
1990 .await
1991 .unwrap(),
1992 );
1993 let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
1994 a.produce(
1995 PartitionKey("p".into()),
1996 Bytes::from_static(b"a-first"),
1997 1,
1998 (),
1999 Durability::Buffered,
2000 )
2001 .await
2002 .unwrap();
2003 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2004 let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
2005 let b_result = tokio::time::timeout(
2006 Duration::from_secs(5),
2007 b.produce(
2008 PartitionKey("p".into()),
2009 Bytes::from_static(b"b-first"),
2010 1,
2011 (),
2012 Durability::Sequenced,
2013 ),
2014 )
2015 .await;
2016 let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
2017 let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
2018 release_tx.send(()).unwrap();
2020 if b_closed.is_err() {
2021 closing_b.await.unwrap();
2022 }
2023 assert!(
2024 b_closed.is_ok(),
2025 "closing B waited for A's blocked manifest"
2026 );
2027 assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
2028 a.flush().await.unwrap();
2029 assert_eq!(
2030 a.produce(
2031 PartitionKey("p".into()),
2032 Bytes::from_static(b"a-second"),
2033 1,
2034 (),
2035 Durability::Sequenced
2036 )
2037 .await
2038 .unwrap()
2039 .base_offset,
2040 Some(1)
2041 );
2042 let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
2043 assert_eq!(
2044 rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
2045 vec![
2046 Bytes::from_static(b"a-first"),
2047 Bytes::from_static(b"a-second")
2048 ]
2049 );
2050 tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
2051 }
2052
2053 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2054 async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
2055 let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
2056 let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
2057 let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
2058 let (upload_release_tx, upload_release_rx) = oneshot::channel();
2059 let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2060 inner: MemoryBlobStore::new(),
2061 entered: manifest_entered_tx,
2062 release: Mutex::new(Some(manifest_release_rx)),
2063 fail: false,
2064 panic_on_commit: true,
2065 })));
2066 *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
2067 let mut config = FlushConfig::default();
2068 config.max_batches = 1;
2069 config.max_inflight_flushes = 4;
2070 config.linger = Duration::ZERO;
2071 config.budget.enabled = false;
2072 let sequencer = Arc::new(
2073 ManifestSequencer::open(store.clone(), "manifest/")
2074 .await
2075 .unwrap(),
2076 );
2077 let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
2078 engine
2079 .produce(
2080 PartitionKey("p".into()),
2081 Bytes::from_static(b"first"),
2082 1,
2083 (),
2084 Durability::Buffered,
2085 )
2086 .await
2087 .unwrap();
2088 manifest_entered_rx
2089 .recv_timeout(Duration::from_secs(5))
2090 .unwrap();
2091 engine
2092 .produce(
2093 PartitionKey("p".into()),
2094 Bytes::from_static(b"second"),
2095 1,
2096 (),
2097 Durability::Buffered,
2098 )
2099 .await
2100 .unwrap();
2101 upload_entered_rx
2102 .recv_timeout(Duration::from_secs(5))
2103 .unwrap();
2104 manifest_release_tx.send(()).unwrap();
2105 tokio::time::timeout(Duration::from_secs(5), async {
2106 while !engine.shared.queue.lock().unwrap().shutdown {
2107 tokio::time::sleep(Duration::from_millis(1)).await;
2108 }
2109 })
2110 .await
2111 .unwrap();
2112 let flush_failed = engine.flush().await.is_err();
2113 let upload_was_owned = upload_release_tx.send(()).is_ok();
2116 tokio::time::timeout(
2117 Duration::from_secs(5),
2118 tokio::task::spawn_blocking(move || drop(engine)),
2119 )
2120 .await
2121 .unwrap()
2122 .unwrap();
2123 assert!(flush_failed);
2124 assert!(upload_was_owned);
2125 assert!(
2126 store
2127 .second_upload_completed
2128 .load(std::sync::atomic::Ordering::SeqCst)
2129 );
2130 assert_eq!(store.list("data/").await.unwrap().len(), 2);
2131 let sequencer = Arc::new(
2132 ManifestSequencer::open(store.clone(), "manifest/")
2133 .await
2134 .unwrap(),
2135 );
2136 let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2137 assert!(
2138 reopened
2139 .fetch(&PartitionKey("p".into()), 0, 100)
2140 .await
2141 .unwrap()
2142 .is_empty()
2143 );
2144 assert_eq!(
2145 reopened
2146 .produce(
2147 PartitionKey("p".into()),
2148 Bytes::from_static(b"after-reopen"),
2149 1,
2150 (),
2151 Durability::Sequenced
2152 )
2153 .await
2154 .unwrap()
2155 .base_offset,
2156 Some(0)
2157 );
2158 let rows = reopened
2159 .fetch(&PartitionKey("p".into()), 0, 100)
2160 .await
2161 .unwrap();
2162 assert_eq!(rows.len(), 1);
2163 assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2164 assert_eq!(store.list("data/").await.unwrap().len(), 3);
2165 tokio::task::spawn_blocking(move || drop(reopened))
2166 .await
2167 .unwrap();
2168 }
2169
2170 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2171 async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2172 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2173 let (release_tx, release_rx) = oneshot::channel();
2174 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2175 inner: MemoryBlobStore::new(),
2176 entered: entered_tx,
2177 release: Mutex::new(Some(release_rx)),
2178 fail: false,
2179 panic_on_commit: false,
2180 });
2181 let sequencer = Arc::new(
2182 ManifestSequencer::open(blob.clone(), "manifest/")
2183 .await
2184 .unwrap(),
2185 );
2186 let mut config = FlushConfig::default();
2187 config.max_batches = 1;
2188 config.max_inflight_flushes = 4;
2189 config.linger = Duration::ZERO;
2190 config.budget.enabled = false;
2191 let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2192 engine
2193 .produce(
2194 PartitionKey("p".into()),
2195 Bytes::from_static(b"first"),
2196 1,
2197 (),
2198 Durability::Buffered,
2199 )
2200 .await
2201 .unwrap();
2202 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2203 engine
2204 .produce(
2205 PartitionKey("p".into()),
2206 Bytes::from_static(b"second"),
2207 1,
2208 (),
2209 Durability::Buffered,
2210 )
2211 .await
2212 .unwrap();
2213 let shared = engine.shared.clone();
2214 let dropping = tokio::task::spawn_blocking(move || drop(engine));
2215 tokio::time::timeout(Duration::from_secs(5), async {
2216 while !shared.queue.lock().unwrap().shutdown {
2217 tokio::time::sleep(Duration::from_millis(1)).await;
2218 }
2219 })
2220 .await
2221 .unwrap();
2222 let premature = dropping.is_finished();
2223 release_tx.send(()).unwrap();
2224 tokio::time::timeout(Duration::from_secs(5), dropping)
2225 .await
2226 .unwrap()
2227 .unwrap();
2228 assert!(!premature, "shutdown returned before manifest durability");
2229 let reopened = Arc::new(
2230 ManifestSequencer::open(blob.clone(), "manifest/")
2231 .await
2232 .unwrap(),
2233 );
2234 let engine = LogEngine::new(blob, reopened, config, "data/");
2235 let rows = engine
2236 .fetch(&PartitionKey("p".into()), 0, 100)
2237 .await
2238 .unwrap();
2239 assert_eq!(
2240 rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2241 vec![b"first".as_slice(), b"second".as_slice()]
2242 );
2243 }
2244
2245 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2246 async fn panicked_committer_closes_admission_and_fails_flush() {
2247 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2248 let (release_tx, release_rx) = oneshot::channel();
2249 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2250 inner: MemoryBlobStore::new(),
2251 entered: entered_tx,
2252 release: Mutex::new(Some(release_rx)),
2253 fail: false,
2254 panic_on_commit: true,
2255 });
2256 let sequencer = Arc::new(
2257 ManifestSequencer::open(blob.clone(), "manifest/")
2258 .await
2259 .unwrap(),
2260 );
2261 let mut config = FlushConfig::default();
2262 config.max_batches = 1;
2263 config.max_inflight_flushes = 4;
2264 config.linger = Duration::ZERO;
2265 config.budget.enabled = false;
2266 let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2267 let first = {
2268 let engine = engine.clone();
2269 tokio::spawn(async move {
2270 engine
2271 .produce(
2272 PartitionKey("p".into()),
2273 Bytes::from_static(b"first"),
2274 1,
2275 (),
2276 Durability::Sequenced,
2277 )
2278 .await
2279 })
2280 };
2281 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2282 engine
2283 .produce(
2284 PartitionKey("p".into()),
2285 Bytes::from_static(b"later"),
2286 1,
2287 (),
2288 Durability::Buffered,
2289 )
2290 .await
2291 .unwrap();
2292 let flushing = {
2293 let engine = engine.clone();
2294 tokio::spawn(async move { engine.flush().await })
2295 };
2296 release_tx.send(()).unwrap();
2297 assert!(
2298 tokio::time::timeout(Duration::from_secs(5), first)
2299 .await
2300 .unwrap()
2301 .unwrap()
2302 .is_err()
2303 );
2304 assert!(
2305 tokio::time::timeout(Duration::from_secs(5), flushing)
2306 .await
2307 .unwrap()
2308 .unwrap()
2309 .is_err()
2310 );
2311 tokio::time::timeout(Duration::from_secs(5), async {
2312 while !engine.shared.queue.lock().unwrap().shutdown {
2313 tokio::time::sleep(Duration::from_millis(1)).await;
2314 }
2315 })
2316 .await
2317 .unwrap();
2318 assert!(
2319 engine
2320 .produce(
2321 PartitionKey("p".into()),
2322 Bytes::from_static(b"rejected"),
2323 1,
2324 (),
2325 Durability::Buffered
2326 )
2327 .await
2328 .is_err()
2329 );
2330 assert!(engine.flush().await.is_err());
2331 }
2332 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2333 async fn committed_reads_continue_during_new_manifest_publication() {
2334 for fail in [false, true] {
2335 let memory = MemoryBlobStore::new();
2336 let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2337 let seed_sequencer = Arc::new(
2338 ManifestSequencer::open(seed_blob.clone(), "manifest/")
2339 .await
2340 .unwrap(),
2341 );
2342 let mut config = FlushConfig::default();
2343 config.max_batches = 1;
2344 config.max_inflight_flushes = 4;
2345 config.linger = Duration::ZERO;
2346 config.budget.enabled = false;
2347 let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2348 seed.produce(
2349 PartitionKey("p".into()),
2350 Bytes::from_static(b"committed"),
2351 1,
2352 (),
2353 Durability::Sequenced,
2354 )
2355 .await
2356 .unwrap();
2357 drop(seed);
2358 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2359 let (release_tx, release_rx) = oneshot::channel();
2360 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2361 inner: memory,
2362 entered: entered_tx,
2363 release: Mutex::new(Some(release_rx)),
2364 fail,
2365 panic_on_commit: false,
2366 });
2367 let sequencer = Arc::new(
2368 ManifestSequencer::open(blob.clone(), "manifest/")
2369 .await
2370 .unwrap(),
2371 );
2372 let engine = Arc::new(LogEngine::new(
2373 blob.clone(),
2374 sequencer.clone(),
2375 config,
2376 "data/",
2377 ));
2378 let producing = {
2379 let engine = engine.clone();
2380 tokio::spawn(async move {
2381 engine
2382 .produce(
2383 PartitionKey("p".into()),
2384 Bytes::from_static(b"new"),
2385 1,
2386 (),
2387 Durability::Sequenced,
2388 )
2389 .await
2390 })
2391 };
2392 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2393 let mut reading = tokio::task::spawn_blocking(move || {
2394 let p = PartitionKey("p".into());
2395 (
2396 sequencer.high_watermark(&p).unwrap(),
2397 sequencer.lookup(&p, 0).unwrap(),
2398 sequencer.snapshot(),
2399 )
2400 });
2401 let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2402 let early_ack = producing.is_finished();
2403 release_tx.send(()).unwrap();
2405 let outcome = producing.await.unwrap();
2406 assert_eq!(outcome.is_err(), fail);
2407 let reads_progressed = observed.is_ok();
2408 let (watermark, entries, snapshot) = match observed {
2409 Ok(result) => result.unwrap(),
2410 Err(_) => reading.await.unwrap(),
2411 };
2412 assert!(!early_ack);
2413 assert!(
2414 reads_progressed,
2415 "durable manifest publication blocked committed index reads"
2416 );
2417 assert_eq!(watermark, 1);
2418 assert_eq!(entries.len(), 1);
2419 assert_eq!(snapshot.manifest_count, 1);
2420 let rows = engine
2421 .fetch(&PartitionKey("p".into()), 0, 100)
2422 .await
2423 .unwrap();
2424 assert_eq!(rows.len(), if fail { 1 } else { 2 });
2425 assert_eq!(rows[0].payload.as_ref(), b"committed");
2426 drop(engine);
2427 let reopened = Arc::new(
2428 ManifestSequencer::open(blob.clone(), "manifest/")
2429 .await
2430 .unwrap(),
2431 );
2432 let engine = LogEngine::new(blob, reopened, config, "data/");
2433 assert_eq!(
2434 engine
2435 .fetch(&PartitionKey("p".into()), 0, 100)
2436 .await
2437 .unwrap()
2438 .len(),
2439 if fail { 1 } else { 2 }
2440 );
2441 }
2442 }
2443 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2444 async fn manifest_mutations_remain_serialized_while_readers_progress() {
2445 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2446 let (release_tx, release_rx) = oneshot::channel();
2447 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2448 inner: MemoryBlobStore::new(),
2449 entered: entered_tx,
2450 release: Mutex::new(Some(release_rx)),
2451 fail: false,
2452 panic_on_commit: false,
2453 });
2454 let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2455 let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2456 tokio::task::spawn_blocking(move || {
2457 sequencer.commit(&[CommitBatch {
2458 partition: PartitionKey("p".into()),
2459 record_count: 1,
2460 location: BatchLocation {
2461 object_id: key.into(),
2462 byte_start: 0,
2463 byte_len: 1,
2464 },
2465 meta: &(),
2466 epoch: 0,
2467 }])
2468 })
2469 };
2470 let first = commit(sequencer.clone(), "data/first");
2471 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2472 let second = commit(sequencer.clone(), "data/second");
2473 let truncating = {
2474 let sequencer = sequencer.clone();
2475 tokio::task::spawn_blocking(move || {
2476 sequencer.truncate_before(&PartitionKey("p".into()), 1)
2477 })
2478 };
2479 let reading = {
2480 let sequencer = sequencer.clone();
2481 tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2482 };
2483 let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2484 release_tx.send(()).unwrap();
2485 assert_eq!(
2486 observed.expect("committed reads blocked").unwrap().unwrap(),
2487 0
2488 );
2489 let first = tokio::time::timeout(Duration::from_secs(5), first)
2490 .await
2491 .unwrap()
2492 .unwrap()
2493 .unwrap();
2494 let second = tokio::time::timeout(Duration::from_secs(5), second)
2495 .await
2496 .unwrap()
2497 .unwrap()
2498 .unwrap();
2499 let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2500 .await
2501 .unwrap()
2502 .unwrap()
2503 .unwrap();
2504 assert_eq!(
2505 first,
2506 vec![CommitOutcome::Assigned {
2507 base_offset: 0,
2508 record_count: 1
2509 }]
2510 );
2511 assert_eq!(
2512 second,
2513 vec![CommitOutcome::Assigned {
2514 base_offset: 1,
2515 record_count: 1
2516 }]
2517 );
2518 assert_eq!(dropped, vec!["data/first".to_string()]);
2519 let p = PartitionKey("p".into());
2520 assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2521 assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2522 let entries = sequencer.lookup(&p, 0).unwrap();
2523 assert_eq!(entries.len(), 1);
2524 assert_eq!(entries[0].location.object_id, "data/second");
2525 assert_eq!(sequencer.snapshot().manifest_count, 2);
2526 }
2527}
2528
2529#[cfg(test)]
2530mod idle_batch_wait_tests {
2531 use super::*;
2532 use std::sync::mpsc;
2533
2534 fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2535 Arc::new(Shared {
2536 queue: Mutex::new(Queue {
2537 items: VecDeque::new(),
2538 bytes: 0,
2539 bytes_in_use: 0,
2540 shutdown: false,
2541 next_seq: 1,
2542 completed_through: 0,
2543 first_failed: None,
2544 force_flush: false,
2545 flush_waiters: Vec::new(),
2546 last_enqueue: None,
2547 oldest_enqueue: None,
2548 }),
2549 cv: Condvar::new(),
2550 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2551 budget: Mutex::new(BudgetRuntime::new(config.budget)),
2552 flush_config: config,
2553 })
2554 }
2555
2556 #[test]
2557 fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2558 for linger in [Duration::ZERO, Duration::from_millis(1)] {
2559 for shutdown in [false, true] {
2560 let config = FlushConfig {
2561 linger,
2562 ..FlushConfig::default()
2563 };
2564 let shared = empty_shared(config);
2565 let (started_tx, started_rx) = mpsc::channel();
2566 let (result_tx, result_rx) = mpsc::channel();
2567 let worker_shared = shared.clone();
2568 let worker = std::thread::spawn(move || {
2569 started_tx.send(()).unwrap();
2570 assert!(
2571 result_tx
2572 .send(take_batch(&worker_shared, config, true))
2573 .is_ok()
2574 );
2575 });
2576 started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2577 let early = result_rx.recv_timeout(Duration::from_millis(100));
2578 let returned_while_idle = early.is_ok();
2579 {
2580 let mut q = shared.queue.lock().unwrap();
2581 if shutdown {
2582 q.shutdown = true;
2583 } else {
2584 let payload = Bytes::from_static(b"first-after-idle");
2585 q.bytes = payload.len();
2586 q.bytes_in_use = payload.len();
2587 q.next_seq = 2;
2588 q.last_enqueue = Some(Instant::now());
2589 q.oldest_enqueue = q.last_enqueue;
2590 q.force_flush = true;
2591 q.items.push_back(Pending {
2592 partition: PartitionKey("p".into()),
2593 record_count: 1,
2594 payload,
2595 meta: (),
2596 epoch: 0,
2597 durability: Durability::Buffered,
2598 responder: None,
2599 seq: 1,
2600 });
2601 }
2602 shared.cv.notify_all();
2603 }
2604 let result = early
2605 .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2606 .unwrap();
2607 worker.join().unwrap();
2608 assert!(
2609 !returned_while_idle,
2610 "idle worker polled with linger={linger:?}"
2611 );
2612 match result {
2613 TakeBatch::Shutdown => assert!(shutdown),
2614 TakeBatch::Batch(items) => {
2615 assert!(!shutdown);
2616 assert_eq!(items.len(), 1);
2617 assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2618 assert_eq!(items[0].seq, 1);
2619 }
2620 TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2621 }
2622 }
2623 }
2624 }
2625}