1use crate::budget::{
4 BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8 BlobStore, CommitBatch, CommitOutcome, ConditionalCommitOutcome, IndexEntry, ObjectLogError,
9 PartitionKey, Sequencer,
10};
11use bytes::Bytes;
12use std::collections::{HashSet, VecDeque};
13use std::sync::{Arc, Condvar, Mutex};
14use std::thread::JoinHandle;
15use std::time::{Duration, Instant};
16use tokio::sync::oneshot;
17use tokio::task::JoinHandle as TokioJoinHandle;
18
19const STORAGE_RETRY_ATTEMPTS: usize = 5;
20const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
21
22#[derive(Clone, Copy, Debug, PartialEq, Eq)]
24pub enum Durability {
25 Buffered,
28 Durable,
31 Sequenced,
34}
35
36#[derive(Clone, Copy, Debug)]
38pub struct FlushConfig {
39 pub max_bytes: usize,
44 pub max_batches: usize,
46 pub linger: Duration,
52 pub max_inflight_flushes: usize,
56 pub max_buffered_bytes: usize,
59 pub budget: BudgetConfig,
61}
62
63impl Default for FlushConfig {
64 fn default() -> Self {
65 Self {
66 max_bytes: 1024 * 1024 * 1024,
68 max_batches: 10_000,
69 linger: Duration::from_millis(50),
70 max_inflight_flushes: 1,
72 max_buffered_bytes: 2 * 1024 * 1024 * 1024,
73 budget: BudgetConfig::default(),
74 }
75 }
76}
77
78#[derive(Clone, Debug, PartialEq, Eq)]
80pub struct AppendOutcome {
81 pub base_offset: Option<i64>,
83 pub last_offset: Option<i64>,
85 pub durable: bool,
87 pub sequenced: bool,
89}
90
91#[derive(Clone, Debug, PartialEq, Eq)]
93pub struct FetchedBatch {
94 pub base_offset: i64,
96 pub record_count: i32,
98 pub payload: Bytes,
100}
101
102#[derive(Clone, Debug, PartialEq, Eq)]
104pub struct BufferStats {
105 pub queued_bytes: usize,
107 pub bytes_in_use: usize,
109 pub queued_batches: usize,
111 pub max_buffered_bytes: usize,
113}
114
115pub type AppendOwner = Arc<dyn std::any::Any + Send + Sync>;
119
120type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
121
122fn retryable_storage_error(err: &ObjectLogError) -> bool {
123 matches!(err, ObjectLogError::StorageUnavailable(_))
124}
125
126async fn retry_delay(attempt: usize) {
127 let multiplier = 1u32 << attempt.min(5);
128 tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
129}
130
131async fn put_chunks_with_retries(
132 blob: &Arc<dyn BlobStore>,
133 key: &str,
134 chunks: Vec<Bytes>,
135) -> Result<(), ObjectLogError> {
136 let mut attempt = 0usize;
138 loop {
139 match blob.put_chunks(key, chunks.clone()).await {
140 Ok(()) => return Ok(()),
141 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
142 retry_delay(attempt).await;
143 attempt += 1;
144 }
145 Err(err) => return Err(err),
146 }
147 }
148}
149
150pub(crate) enum RangeRead {
151 Legacy,
152 Exact,
153}
154pub(crate) async fn get_range_with_retries(
155 blob: &Arc<dyn BlobStore>,
156 key: &str,
157 range: std::ops::Range<u64>,
158 mode: RangeRead,
159) -> Result<Option<Bytes>, ObjectLogError> {
160 let mut attempt = 0usize;
161 loop {
162 let result = match mode {
163 RangeRead::Legacy => blob.get_range(key, range.clone()).await,
164 RangeRead::Exact => blob.get_range_exact(key, range.clone()).await,
165 };
166 match result {
167 Ok(bytes) => return Ok(bytes),
168 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
169 retry_delay(attempt).await;
170 attempt += 1;
171 }
172 Err(err) => return Err(err),
173 }
174 }
175}
176
177struct Pending<M> {
178 partition: PartitionKey,
179 record_count: i32,
180 payload: Bytes,
181 meta: M,
182 epoch: u64,
184 durability: Durability,
185 responder: Option<Responder>,
186 seq: u64,
188 owner: Option<AppendOwner>,
189 expected_head: Option<i64>,
190}
191
192struct FlushWork<M> {
193 batch: Vec<Pending<M>>,
194 locations: Vec<BatchLocation>,
195 responders: Vec<(Durability, Option<Responder>)>,
196 bytes: usize,
197 max_seq: u64,
199 data_objects: usize,
201 put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
202 put_started: Instant,
203 put_result: Option<Result<Duration, ObjectLogError>>,
204}
205
206struct CommitJob {
207 task: TokioJoinHandle<usize>,
208 first_seq: u64,
209}
210
211enum TakeBatch<M> {
212 Batch(Vec<Pending<M>>),
213 Empty,
214 Shutdown,
215}
216
217type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
218
219struct Queue<M> {
220 items: VecDeque<Pending<M>>,
221 bytes: usize,
222 bytes_in_use: usize,
223 shutdown: bool,
224 next_seq: u64,
226 completed_through: u64,
228 first_failed: Option<(u64, ObjectLogError)>,
230 force_flush: bool,
232 flush_waiters: Vec<FlushWaiter>,
234 last_enqueue: Option<Instant>,
236 oldest_enqueue: Option<Instant>,
238}
239
240struct Shared<M> {
241 queue: Mutex<Queue<M>>,
242 cv: Condvar,
243 max_buffered_bytes: usize,
244 budget: Mutex<BudgetRuntime>,
246 flush_config: FlushConfig,
248}
249
250pub struct LogEngine<S: Sequencer> {
259 shared: Arc<Shared<S::Meta>>,
260 blob: Arc<dyn BlobStore>,
261 sequencer: Arc<S>,
262 key_prefix: String,
265 writer_id: String,
266 flush_thread: Option<JoinHandle<()>>,
267}
268
269impl<S> LogEngine<S>
270where
271 S: Sequencer + 'static,
272 S::Meta: Send + 'static,
273{
274 pub fn new(
278 blob: Arc<dyn BlobStore>,
279 sequencer: Arc<S>,
280 config: FlushConfig,
281 key_prefix: impl Into<String>,
282 ) -> Self {
283 Self::new_with_writer(blob, sequencer, config, key_prefix, "")
284 }
285
286 pub fn new_with_writer(
290 blob: Arc<dyn BlobStore>,
291 sequencer: Arc<S>,
292 config: FlushConfig,
293 key_prefix: impl Into<String>,
294 writer_id: impl Into<String>,
295 ) -> Self {
296 if let Err(msg) = config.budget.validate() {
297 panic!("invalid FlushConfig.budget: {msg}");
298 }
299 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
300 eprintln!("object-log flush config: {config:?}");
301 }
302 let budget_rt = BudgetRuntime::new(config.budget);
303 let shared = Arc::new(Shared {
304 queue: Mutex::new(Queue {
305 items: VecDeque::new(),
306 bytes: 0,
307 bytes_in_use: 0,
308 shutdown: false,
309 next_seq: 1, completed_through: 0,
311 first_failed: None,
312 force_flush: false,
313 flush_waiters: Vec::new(),
314 last_enqueue: None,
315 oldest_enqueue: None,
316 }),
317 cv: Condvar::new(),
318 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
319 budget: Mutex::new(budget_rt),
320 flush_config: config,
321 });
322 let key_prefix = key_prefix.into();
323 let writer_id = writer_id.into();
324 let flush_thread = {
325 let shared = Arc::clone(&shared);
326 let blob = Arc::clone(&blob);
327 let sequencer = Arc::clone(&sequencer);
328 let prefix = key_prefix.clone();
329 let writer_id = writer_id.clone();
330 std::thread::Builder::new()
331 .name("object-log-flush".into())
332 .spawn(move || flush_loop(shared, blob, sequencer, config, prefix, writer_id))
333 .expect("spawn flush thread")
334 };
335 Self {
336 writer_id,
337 shared,
338 blob,
339 sequencer,
340 key_prefix,
341 flush_thread: Some(flush_thread),
342 }
343 }
344
345 pub fn data_prefix(&self) -> &str {
347 &self.key_prefix
348 }
349
350 pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
352 &self.blob
353 }
354
355 pub fn sequencer(&self) -> &Arc<S> {
357 &self.sequencer
358 }
359
360 pub fn writer_id(&self) -> &str {
362 &self.writer_id
363 }
364
365 pub async fn produce(
370 &self,
371 partition: PartitionKey,
372 payload: Bytes,
373 record_count: i32,
374 meta: S::Meta,
375 durability: Durability,
376 ) -> Result<AppendOutcome, ObjectLogError> {
377 self.produce_at_epoch(partition, payload, record_count, meta, durability, 0)
378 .await
379 }
380
381 pub async fn produce_at_epoch(
383 &self,
384 partition: PartitionKey,
385 payload: Bytes,
386 record_count: i32,
387 meta: S::Meta,
388 durability: Durability,
389 epoch: u64,
390 ) -> Result<AppendOutcome, ObjectLogError> {
391 self.produce_at_epoch_with_owner(
392 partition,
393 payload,
394 record_count,
395 meta,
396 durability,
397 epoch,
398 None,
399 None,
400 )
401 .await
402 }
403
404 #[allow(clippy::too_many_arguments)]
408 pub async fn produce_owned_at_epoch(
409 &self,
410 partition: PartitionKey,
411 payload: Bytes,
412 record_count: i32,
413 meta: S::Meta,
414 durability: Durability,
415 epoch: u64,
416 owner: AppendOwner,
417 ) -> Result<AppendOutcome, ObjectLogError> {
418 self.produce_at_epoch_with_owner(
419 partition,
420 payload,
421 record_count,
422 meta,
423 durability,
424 epoch,
425 Some(owner),
426 None,
427 )
428 .await
429 }
430
431 #[allow(clippy::too_many_arguments)]
433 pub async fn produce_conditional_at_epoch(
434 &self,
435 partition: PartitionKey,
436 payload: Bytes,
437 record_count: i32,
438 meta: S::Meta,
439 epoch: u64,
440 expected_head: i64,
441 owner: AppendOwner,
442 ) -> Result<AppendOutcome, ObjectLogError> {
443 self.produce_at_epoch_with_owner(
444 partition,
445 payload,
446 record_count,
447 meta,
448 Durability::Sequenced,
449 epoch,
450 Some(owner),
451 Some(expected_head),
452 )
453 .await
454 }
455
456 #[allow(clippy::too_many_arguments)]
457 async fn produce_at_epoch_with_owner(
458 &self,
459 partition: PartitionKey,
460 payload: Bytes,
461 record_count: i32,
462 meta: S::Meta,
463 durability: Durability,
464 epoch: u64,
465 owner: Option<AppendOwner>,
466 expected_head: Option<i64>,
467 ) -> Result<AppendOutcome, ObjectLogError> {
468 if let Some(head) = expected_head {
469 if head < 0 {
470 return Err(ObjectLogError::InvalidBatch(
471 "negative expected head".into(),
472 ));
473 }
474 if !self.sequencer.supports_conditional_commit() {
475 return Err(ObjectLogError::InvalidConfig(
476 "conditional commit unsupported".into(),
477 ));
478 }
479 }
480 if record_count <= 0 {
481 return Err(ObjectLogError::InvalidBatch(
482 "record_count must be > 0".into(),
483 ));
484 }
485 if matches!(durability, Durability::Buffered) {
486 self.enqueue(Pending {
487 partition,
488 record_count,
489 payload,
490 meta,
491 epoch,
492 durability,
493 responder: None,
494 owner,
495 expected_head,
496 seq: 0, })?;
498 return Ok(AppendOutcome {
499 base_offset: None,
500 last_offset: None,
501 durable: false,
502 sequenced: false,
503 });
504 }
505 {
507 let mut budget = self.shared.budget.lock().expect("poisoned");
508 if budget.config.enabled
509 && budget.config.mode == BudgetMode::FailClosed
510 && !budget.reserve_for_fail_closed(Instant::now())
511 {
512 return Err(ObjectLogError::BudgetExceeded(
513 "insufficient durable-ops budget to admit produce".into(),
514 ));
515 }
516 }
517 if self.shared.flush_config.budget.enabled
519 && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
520 {
521 let timeout = self.shared.flush_config.budget.admission_timeout;
522 let deadline = Instant::now() + timeout;
523 loop {
524 let mut budget = self.shared.budget.lock().expect("poisoned");
525 if budget.can_admit_now(Instant::now()) {
526 break;
527 }
528 drop(budget);
529 if Instant::now() >= deadline {
530 return Err(ObjectLogError::BudgetExceeded(
531 "timed out waiting for durable-ops budget".into(),
532 ));
533 }
534 std::thread::sleep(Duration::from_millis(1));
535 }
536 }
537 let (tx, rx) = oneshot::channel();
538 self.enqueue(Pending {
539 partition,
540 record_count,
541 payload,
542 meta,
543 epoch,
544 durability,
545 responder: Some(tx),
546 owner,
547 expected_head,
548 seq: 0,
549 })?;
550 rx.await
551 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
552 }
553
554 pub async fn flush(&self) -> Result<(), ObjectLogError> {
563 let rx = {
564 let mut q = self.shared.queue.lock().expect("poisoned");
565 if q.shutdown {
566 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
567 }
568 if q.next_seq == 1 {
569 return Ok(());
571 }
572 let barrier = q.next_seq - 1;
573 if let Some((sequence, error)) = &q.first_failed
574 && *sequence <= barrier
575 {
576 return Err(error.clone());
577 }
578 if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
579 return Ok(());
580 }
581 let (tx, rx) = oneshot::channel();
582 q.force_flush = true;
583 q.flush_waiters.push((barrier, tx));
584 self.shared.cv.notify_all();
585 rx
586 };
587 rx.await
588 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
589 }
590
591 fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
592 let item_len = item.payload.len();
593 let max_buffered_bytes = self.shared.max_buffered_bytes;
594 let shared = Arc::clone(&self.shared);
595 let mut q = shared.queue.lock().expect("poisoned");
596 while !q.shutdown
597 && q.bytes_in_use > 0
598 && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
599 {
600 q = shared.cv.wait(q).expect("poisoned");
601 }
602 if q.shutdown {
603 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
604 }
605 let now = Instant::now();
606 item.seq = q.next_seq;
607 q.next_seq = q.next_seq.saturating_add(1);
608 q.bytes += item_len;
609 q.bytes_in_use += item_len;
610 q.last_enqueue = Some(now);
611 if q.oldest_enqueue.is_none() {
612 q.oldest_enqueue = Some(now);
613 }
614 q.items.push_back(item);
615 shared.cv.notify_all();
616 Ok(())
617 }
618
619 pub async fn fetch(
621 &self,
622 partition: &PartitionKey,
623 offset: i64,
624 max_bytes: usize,
625 ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
626 let entries = self.sequencer.lookup(partition, offset)?;
627 let mut out = Vec::new();
628 let mut total = 0usize;
629 for e in entries {
630 if total >= max_bytes && !out.is_empty() {
631 break;
632 }
633 let batch = self.load_entry(&e).await?;
634 total += batch.payload.len();
635 out.push(batch);
636 }
637 Ok(out)
638 }
639
640 pub async fn fetch_stream<F>(
647 &self,
648 partition: &PartitionKey,
649 offset: i64,
650 mut visit: F,
651 ) -> Result<(), ObjectLogError>
652 where
653 F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
654 {
655 let entries = self.sequencer.lookup(partition, offset)?;
656 for e in entries {
657 visit(self.load_entry(&e).await?)?;
658 }
659 Ok(())
660 }
661
662 async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
663 let start = e.location.byte_start as u64;
664 let end = start + e.location.byte_len as u64;
665 let bytes = get_range_with_retries(
666 &self.blob,
667 &e.location.object_id,
668 start..end,
669 RangeRead::Legacy,
670 )
671 .await?
672 .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
673 Ok(FetchedBatch {
674 base_offset: e.base_offset,
675 record_count: e.record_count,
676 payload: bytes,
677 })
678 }
679
680 pub async fn truncate_before(
683 &self,
684 partition: &PartitionKey,
685 offset: i64,
686 ) -> Result<(), ObjectLogError> {
687 let dead = self.sequencer.truncate_before(partition, offset)?;
688 for object_id in dead {
689 self.blob.delete(&object_id).await?;
690 }
691 Ok(())
692 }
693
694 pub async fn reap_orphans(
704 &self,
705 live: &HashSet<String>,
706 ) -> Result<Vec<String>, ObjectLogError> {
707 reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
708 }
709
710 pub fn buffer_stats(&self) -> BufferStats {
712 let q = self.shared.queue.lock().expect("poisoned");
713 BufferStats {
714 queued_bytes: q.bytes,
715 bytes_in_use: q.bytes_in_use,
716 queued_batches: q.items.len(),
717 max_buffered_bytes: self.shared.max_buffered_bytes,
718 }
719 }
720
721 pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
723 let now = Instant::now();
724 let mut budget = self.shared.budget.lock().expect("poisoned");
725 budget.refill(now);
726 let effective = budget.effective_budget_per_sec;
727 let reason = if !budget.config.enabled {
728 EffectiveReason::Disabled
729 } else if budget.config.budget_per_sec_cap.is_some()
730 && budget
731 .config
732 .budget_per_sec_cap
733 .is_some_and(|c| (effective - c).abs() < 1e-9)
734 {
735 EffectiveReason::ConfigCap
736 } else if budget.ongoing_capacity.is_some() {
737 EffectiveReason::Ongoing
738 } else if budget.startup_capacity.is_some() {
739 EffectiveReason::StartupProbe
740 } else {
741 EffectiveReason::DefaultCapacity
742 };
743 let (queued, last_enq) = {
744 let q = self.shared.queue.lock().expect("poisoned");
745 (q.bytes, q.last_enqueue)
746 };
747 let early = budget.allow_early_flush(now, queued, last_enq);
748 let max_linger = self.shared.flush_config.linger;
749 let eff_linger = if !budget.config.enabled {
750 max_linger
751 } else if early || max_linger.is_zero() {
752 Duration::ZERO
753 } else {
754 max_linger
755 };
756 PipelineSnapshot {
757 budget_per_sec: EffectiveKnob {
758 configured: budget.config.budget_per_sec_cap,
759 startup_measured: budget.startup_capacity,
760 ongoing_measured: budget.ongoing_capacity,
761 effective,
762 reason,
763 },
764 effective_linger_ms: EffectiveKnob {
765 configured: Some(max_linger.as_millis() as u64),
766 startup_measured: None,
767 ongoing_measured: None,
768 effective: eff_linger.as_millis() as u64,
769 reason: if early {
770 EffectiveReason::Ongoing
771 } else {
772 EffectiveReason::Configured
773 },
774 },
775 max_linger_ms: max_linger.as_millis() as u64,
776 token_fill_ratio: budget.fill_ratio(),
777 media_ops_total: budget.media_ops_total,
778 overdraft_total: budget.overdraft_total,
779 flushes_total: budget.flushes_total,
780 undersized_deadline_flushes: budget.undersized_deadline_flushes,
781 predicted_media_ops: budget.predicted_media_ops,
782 budget_enabled: budget.config.enabled,
783 budget_mode: budget.config.mode,
784 }
785 }
786}
787
788pub async fn reap_orphans(
794 blob: &dyn BlobStore,
795 data_prefix: &str,
796 live: &HashSet<String>,
797) -> Result<Vec<String>, ObjectLogError> {
798 let keys = blob.list(data_prefix).await?;
799 let mut deleted = Vec::new();
800 for key in keys {
801 if live.contains(&key) {
802 continue;
803 }
804 blob.delete(&key).await?;
805 deleted.push(key);
806 }
807 Ok(deleted)
808}
809
810impl<S: Sequencer> Drop for LogEngine<S> {
811 fn drop(&mut self) {
812 {
813 let mut q = self.shared.queue.lock().expect("poisoned");
814 q.shutdown = true;
815 }
816 self.shared.cv.notify_all();
817 if let Some(t) = self.flush_thread.take() {
818 let _ = t.join();
819 }
820 }
821}
822
823async fn recover_data_object_counter(
829 blob: &dyn BlobStore,
830 prefix: &str,
831 writer_id: &str,
832) -> Result<u64, ObjectLogError> {
833 let keys = blob.list(prefix).await?;
834 let mut max = 0u64;
835 let writer_prefix = format!("{prefix}{writer_id}/");
836 for key in keys {
837 let Some(suffix) = key.strip_prefix(prefix) else {
838 continue;
839 };
840 if writer_id.is_empty() {
841 if let Ok(n) = suffix.parse::<u64>() {
842 max = max.max(n);
843 }
844 continue;
845 }
846 let Some(rest) = key.strip_prefix(&writer_prefix) else {
847 continue;
848 };
849 if let Some(n) = rest
850 .rsplit('/')
851 .next()
852 .and_then(|tail| tail.parse::<u64>().ok())
853 {
854 max = max.max(n);
855 }
856 }
857 Ok(max)
858}
859
860fn data_object_key(prefix: &str, writer_id: &str, epoch: u64, counter: u64) -> String {
861 if writer_id.is_empty() {
862 format!("{prefix}{counter:020}")
863 } else {
864 format!("{prefix}{writer_id}/{epoch:020}/{counter:020}")
865 }
866}
867
868fn flush_loop<S>(
869 shared: Arc<Shared<S::Meta>>,
870 blob: Arc<dyn BlobStore>,
871 sequencer: Arc<S>,
872 config: FlushConfig,
873 prefix: String,
874 writer_id: String,
875) where
876 S: Sequencer + 'static,
877 S::Meta: Send + 'static,
878{
879 let max_inflight = config.max_inflight_flushes.max(1);
880 let commit_group_limit = if sequencer.supports_multi_object_commit() {
881 max_inflight
882 } else {
883 1
884 };
885 let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
886 .ok()
887 .and_then(|v| v.parse::<usize>().ok())
888 .filter(|v| *v > 0)
889 .unwrap_or_else(|| max_inflight.min(8));
890 let rt = tokio::runtime::Builder::new_multi_thread()
891 .worker_threads(worker_threads)
892 .enable_all()
893 .build()
894 .expect("flush runtime");
895 let mut counter = match rt.block_on(recover_data_object_counter(
899 blob.as_ref(),
900 &prefix,
901 &writer_id,
902 )) {
903 Ok(counter) => counter,
904 Err(error) => {
905 let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
908 return;
909 }
910 };
911 let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
912 let mut active_puts = 0usize;
913 let mut committing: Option<CommitJob> = None;
914 let mut shutdown = false;
915
916 loop {
917 while !shutdown && active_puts < max_inflight {
918 let wait_for_more = pending.is_empty() && committing.is_none();
921 match take_batch(&shared, config, wait_for_more) {
922 TakeBatch::Batch(batch) => {
923 counter += 1;
924 let concurrent = max_inflight > 1;
925 let work = start_flush_work(
926 &rt, &blob, &prefix, &writer_id, counter, batch, concurrent,
927 );
928 if concurrent {
929 active_puts += 1;
930 }
931 pending.push_back(work);
932 }
933 TakeBatch::Empty => break,
934 TakeBatch::Shutdown => shutdown = true,
935 }
936 }
937
938 let mut made_progress = false;
939 for work in pending.iter_mut() {
940 let Some(put) = work.put.as_ref() else {
941 continue;
942 };
943 if !put.is_finished() {
944 continue;
945 }
946 let put = work.put.take().expect("put handle exists");
947 let elapsed = work.put_started.elapsed();
948 work.put_result = Some(match rt.block_on(put) {
949 Ok(Ok(())) => Ok(elapsed),
950 Ok(Err(e)) => Err(e),
951 Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
952 "flush task failed: {e}"
953 ))),
954 });
955 active_puts = active_puts.saturating_sub(1);
956 made_progress = true;
957 }
958
959 if committing
960 .as_ref()
961 .is_some_and(|job| job.task.is_finished())
962 {
963 let job = committing.take().expect("completed commit exists");
964 match rt.block_on(job.task) {
965 Ok(released) => {
966 let mut q = shared.queue.lock().expect("poisoned");
967 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
968 shared.cv.notify_all();
969 }
970 Err(error) => {
971 let uploads = fail_engine_and_take_pending_uploads(
972 &shared,
973 &mut pending,
974 job.first_seq,
975 ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
976 );
977 for upload in uploads {
981 let _ = rt.block_on(upload);
982 }
983 return;
984 }
985 }
986 made_progress = true;
987 }
988
989 if committing.is_none()
990 && let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit)
991 {
992 if max_inflight == 1 {
993 let released = finish_flush_work(&shared, &blob, &sequencer, work);
995 let mut q = shared.queue.lock().expect("poisoned");
996 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
997 shared.cv.notify_all();
998 } else {
999 let first_seq = work
1002 .batch
1003 .iter()
1004 .map(|p| p.seq)
1005 .min()
1006 .unwrap_or(work.max_seq);
1007 let shared = shared.clone();
1008 let blob = blob.clone();
1009 let sequencer = sequencer.clone();
1010 committing = Some(CommitJob {
1011 task: rt.spawn_blocking(move || {
1012 finish_flush_work(&shared, &blob, &sequencer, work)
1013 }),
1014 first_seq,
1015 });
1016 }
1017 made_progress = true;
1018 }
1019
1020 if pending.is_empty() && committing.is_none() {
1021 {
1022 let mut q = shared.queue.lock().expect("poisoned");
1023 if q.items.is_empty() && q.bytes_in_use == 0 {
1024 notify_flush_waiters(&mut q);
1027 }
1028 }
1029 shared.cv.notify_all();
1030 if shutdown {
1031 let mut q = shared.queue.lock().expect("poisoned");
1033 for (_, tx) in q.flush_waiters.drain(..) {
1034 let _ = tx.send(Err(ObjectLogError::Sequencer(
1035 "engine shutting down".into(),
1036 )));
1037 }
1038 return;
1039 }
1040 continue;
1041 }
1042
1043 if !made_progress {
1044 std::thread::sleep(Duration::from_millis(1));
1045 }
1046 }
1047}
1048
1049fn fail_engine_and_take_pending_uploads<M>(
1052 shared: &Arc<Shared<M>>,
1053 pending: &mut VecDeque<FlushWork<M>>,
1054 first_seq: u64,
1055 error: ObjectLogError,
1056) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
1057 let mut uploads = Vec::new();
1058 for mut work in pending.drain(..) {
1059 if let Some(put) = work.put.take() {
1060 uploads.push(put);
1061 }
1062 send_storage_error(&mut work.responders, error.clone());
1063 }
1064 let mut q = shared
1066 .queue
1067 .lock()
1068 .unwrap_or_else(|poisoned| poisoned.into_inner());
1069 q.shutdown = true;
1070 q.first_failed.get_or_insert((first_seq, error.clone()));
1071 let rejected = std::mem::take(&mut q.items);
1072 q.bytes = 0;
1073 for (_, tx) in q.flush_waiters.drain(..) {
1076 let _ = tx.send(Err(error.clone()));
1077 }
1078 q.force_flush = false;
1079 shared.cv.notify_all();
1080 drop(q);
1081 for mut item in rejected {
1084 if let Some(tx) = item.responder.take() {
1085 let _ = tx.send(Err(error.clone()));
1086 }
1087 }
1088 uploads
1089}
1090
1091fn take_ready_commit_group<M>(
1094 pending: &mut VecDeque<FlushWork<M>>,
1095 max_objects: usize,
1096) -> Option<FlushWork<M>> {
1097 pending.front()?.put_result.as_ref()?;
1098 let mut group = pending.pop_front().expect("ready front exists");
1099 if group.put_result.as_ref().is_some_and(Result::is_err) {
1100 return Some(group);
1101 }
1102 while group.data_objects < max_objects.max(1) {
1103 let Some(next) = pending.front() else { break };
1104 if !next.put_result.as_ref().is_some_and(Result::is_ok)
1105 || group.data_objects.saturating_add(next.data_objects) > max_objects
1106 {
1107 break;
1108 }
1109 let mut next = pending.pop_front().expect("ready successor exists");
1110 group.batch.append(&mut next.batch);
1111 group.locations.append(&mut next.locations);
1112 group.responders.append(&mut next.responders);
1113 group.bytes = group
1114 .bytes
1115 .checked_add(next.bytes)
1116 .expect("bounded pending bytes");
1117 group.max_seq = group.max_seq.max(next.max_seq);
1118 group.data_objects += next.data_objects;
1119 let next_elapsed = next.put_result.take().unwrap().unwrap();
1120 let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
1121 *elapsed = (*elapsed).max(next_elapsed);
1122 }
1123 Some(group)
1124}
1125
1126fn effective_linger<M>(
1130 shared: &Shared<M>,
1131 config: FlushConfig,
1132 queued_bytes: usize,
1133 last_enqueue: Option<Instant>,
1134 oldest_enqueue: Option<Instant>,
1135) -> Duration {
1136 if config.linger.is_zero() {
1137 return Duration::ZERO;
1138 }
1139 let now = Instant::now();
1140 if config.budget.enabled {
1141 let mut budget = shared.budget.lock().expect("poisoned");
1142 budget.refill(now);
1143 if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
1144 budget.note_early_flush(now);
1145 return Duration::ZERO;
1146 }
1147 }
1148 if let Some(oldest) = oldest_enqueue {
1150 let deadline = oldest + config.linger;
1151 if now >= deadline {
1152 return Duration::ZERO;
1153 }
1154 let until_deadline = deadline.saturating_duration_since(now);
1155 if config.budget.enabled
1157 && !config.budget.early_flush_idle.is_zero()
1158 && let Some(last) = last_enqueue
1159 {
1160 let idle_at = last + config.budget.early_flush_idle;
1161 if idle_at > now && idle_at < deadline {
1162 return idle_at.saturating_duration_since(now);
1163 }
1164 }
1165 return until_deadline;
1166 }
1167 config.linger
1168}
1169
1170fn take_batch<M>(
1171 shared: &Arc<Shared<M>>,
1172 config: FlushConfig,
1173 wait_when_empty: bool,
1174) -> TakeBatch<M> {
1175 let mut q = shared.queue.lock().expect("poisoned");
1176 loop {
1177 if q.items.is_empty() {
1178 if q.shutdown {
1179 return TakeBatch::Shutdown;
1180 }
1181 if !wait_when_empty {
1182 return TakeBatch::Empty;
1183 }
1184 q = shared.cv.wait(q).expect("poisoned");
1188 continue;
1189 }
1190
1191 let linger = {
1192 let queued = q.bytes;
1193 let last_enq = q.last_enqueue;
1194 let oldest = q.oldest_enqueue;
1195 drop(q);
1196 let l = effective_linger(shared, config, queued, last_enq, oldest);
1197 q = shared.queue.lock().expect("poisoned");
1198 l
1199 };
1200
1201 let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1202 let force = q.force_flush;
1203 let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1204 if triggered {
1205 if !size_trigger && !q.shutdown && !q.items.is_empty() {
1206 let mut budget = shared.budget.lock().expect("poisoned");
1207 if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1208 budget.undersized_deadline_flushes += 1;
1209 }
1210 }
1211 break;
1212 }
1213
1214 let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1217 q = guard;
1218 }
1219
1220 let force_drain = q.force_flush;
1221 let mut items = Vec::new();
1222 let mut bytes = 0usize;
1223 while let Some(item) = q.items.pop_front() {
1226 q.bytes = q.bytes.saturating_sub(item.payload.len());
1227 bytes += item.payload.len();
1228 items.push(item);
1229 if bytes >= config.max_bytes {
1230 break;
1231 }
1232 if !force_drain && items.len() >= config.max_batches {
1233 break;
1234 }
1235 }
1236 q.oldest_enqueue = if q.items.is_empty() {
1237 None
1238 } else {
1239 Some(Instant::now())
1241 };
1242 if force_drain {
1244 q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1245 }
1246 if items.is_empty() {
1247 TakeBatch::Empty
1248 } else {
1249 TakeBatch::Batch(items)
1250 }
1251}
1252
1253fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1254 let done = q.completed_through;
1255 let mut i = 0;
1256 while i < q.flush_waiters.len() {
1257 let barrier = q.flush_waiters[i].0;
1258 let failure = q
1259 .first_failed
1260 .as_ref()
1261 .filter(|(sequence, _)| *sequence <= barrier)
1262 .map(|(_, error)| error.clone());
1263 if barrier <= done || failure.is_some() {
1264 let (_, tx) = q.flush_waiters.swap_remove(i);
1265 let _ = tx.send(failure.map_or(Ok(()), Err));
1266 } else {
1267 i += 1;
1268 }
1269 }
1270 if q.flush_waiters.is_empty() {
1271 q.force_flush = false;
1272 } else if q.items.is_empty() && q.bytes_in_use == 0 {
1273 q.force_flush = true;
1275 } else {
1276 q.force_flush = true;
1277 }
1278}
1279
1280fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1281 for (_, tx) in responders.iter_mut() {
1282 if let Some(tx) = tx.take() {
1283 let _ = tx.send(Err(err.clone()));
1284 }
1285 }
1286}
1287
1288fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1289 for (durability, tx) in responders.iter_mut() {
1290 if *durability == Durability::Durable
1291 && let Some(tx) = tx.take()
1292 {
1293 let _ = tx.send(Ok(AppendOutcome {
1294 base_offset: None,
1295 last_offset: None,
1296 durable: true,
1297 sequenced: false,
1298 }));
1299 }
1300 }
1301}
1302
1303fn prepare_flush_work<M>(
1304 prefix: &str,
1305 writer_id: &str,
1306 counter: u64,
1307 mut batch: Vec<Pending<M>>,
1308) -> (FlushWork<M>, String, Vec<Bytes>) {
1309 let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1310 let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1311 let epoch = batch.first().map(|pending| pending.epoch).unwrap_or(0);
1312 let key = data_object_key(prefix, writer_id, epoch, counter);
1313 let mut offset = 0usize;
1314 for p in &batch {
1315 let start = offset as u32;
1316 offset += p.payload.len();
1317 chunks.push(p.payload.clone());
1318 locations.push(BatchLocation {
1319 object_id: key.clone(),
1320 byte_start: start,
1321 byte_len: p.payload.len() as u32,
1322 });
1323 }
1324 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1325 eprintln!(
1326 "object-log seal: key={key} batches={} bytes={offset}",
1327 batch.len()
1328 );
1329 }
1330 let responders: Vec<(Durability, Option<Responder>)> = batch
1331 .iter_mut()
1332 .map(|p| (p.durability, p.responder.take()))
1333 .collect();
1334 let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1335 let work = FlushWork {
1336 batch,
1337 locations,
1338 responders,
1339 bytes: offset,
1340 max_seq,
1341 data_objects: 1,
1342 put: None,
1343 put_started: Instant::now(),
1344 put_result: None,
1345 };
1346 (work, key, chunks)
1347}
1348
1349fn start_flush_work<M>(
1355 rt: &tokio::runtime::Runtime,
1356 blob: &Arc<dyn BlobStore>,
1357 prefix: &str,
1358 writer_id: &str,
1359 counter: u64,
1360 batch: Vec<Pending<M>>,
1361 concurrent: bool,
1362) -> FlushWork<M> {
1363 let (mut work, key, chunks) = prepare_flush_work(prefix, writer_id, counter, batch);
1364 let owners = work
1365 .batch
1366 .iter()
1367 .filter_map(|pending| pending.owner.clone())
1368 .collect();
1369 let upload = OwnedUpload {
1370 blob: Arc::clone(blob),
1371 key,
1372 chunks,
1373 _owners: owners,
1374 };
1375 let _ = blob.take_media_op_stats();
1376 work.put_started = Instant::now();
1377 if concurrent {
1378 work.put = Some(rt.spawn(upload.execute()));
1379 } else {
1380 let put_result = rt.block_on(upload.execute());
1381 let elapsed = work.put_started.elapsed();
1382 work.put_result = Some(match put_result {
1383 Ok(()) => Ok(elapsed),
1384 Err(e) => Err(e),
1385 });
1386 }
1387 work
1388}
1389
1390struct OwnedUpload {
1393 blob: Arc<dyn BlobStore>,
1394 key: String,
1395 chunks: Vec<Bytes>,
1396 _owners: Vec<AppendOwner>,
1397}
1398impl OwnedUpload {
1399 async fn execute(mut self) -> Result<(), ObjectLogError> {
1400 let chunks = std::mem::take(&mut self.chunks);
1401 let result = put_chunks_with_retries(&self.blob, &self.key, chunks).await;
1402 drop(self);
1403 result
1404 }
1405}
1406
1407enum CommitResults {
1408 Legacy(Vec<CommitOutcome>),
1409 Conditional(Vec<ConditionalCommitOutcome>),
1410}
1411impl CommitResults {
1412 fn len(&self) -> usize {
1413 match self {
1414 Self::Legacy(rows) => rows.len(),
1415 Self::Conditional(rows) => rows.len(),
1416 }
1417 }
1418 fn into_iter(self) -> CommitResultIter {
1419 match self {
1420 Self::Legacy(rows) => CommitResultIter::Legacy(rows.into_iter()),
1421 Self::Conditional(rows) => CommitResultIter::Conditional(rows.into_iter()),
1422 }
1423 }
1424}
1425enum CommitResultIter {
1426 Legacy(std::vec::IntoIter<CommitOutcome>),
1427 Conditional(std::vec::IntoIter<ConditionalCommitOutcome>),
1428}
1429impl Iterator for CommitResultIter {
1430 type Item = ConditionalCommitOutcome;
1431 fn next(&mut self) -> Option<Self::Item> {
1432 match self {
1433 Self::Legacy(rows) => rows.next().map(ConditionalCommitOutcome::Committed),
1434 Self::Conditional(rows) => rows.next(),
1435 }
1436 }
1437}
1438
1439fn finish_flush_work<S, M>(
1440 shared: &Arc<Shared<M>>,
1441 blob: &Arc<dyn BlobStore>,
1442 sequencer: &Arc<S>,
1443 mut work: FlushWork<S::Meta>,
1444) -> usize
1445where
1446 S: Sequencer + 'static,
1447 S::Meta: Send + 'static,
1448{
1449 let release_bytes = work.bytes;
1450 let first_seq = work
1451 .batch
1452 .iter()
1453 .map(|pending| pending.seq)
1454 .min()
1455 .unwrap_or(work.max_seq);
1456 let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1460 let put_elapsed = match work.put_result.take().expect("put result is ready") {
1461 Ok(elapsed) => elapsed,
1462 Err(e) => {
1463 send_storage_error(&mut work.responders, e.clone());
1464 let mut q = shared.queue.lock().expect("poisoned");
1465 let done = work.max_seq;
1467 if done >= q.completed_through {
1468 q.completed_through = done;
1469 }
1470 q.first_failed.get_or_insert((first_seq, e));
1471 notify_flush_waiters(&mut q);
1472 shared.cv.notify_all();
1473 return release_bytes;
1474 }
1475 };
1476
1477 let mut media_ops = blob
1479 .take_media_op_stats()
1480 .map(|s| s.media_ops)
1481 .unwrap_or(work.data_objects as u64); send_durable_acks(&mut work.responders);
1485
1486 let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1488 .batch
1489 .iter()
1490 .zip(work.locations.iter())
1491 .map(|(p, loc)| CommitBatch {
1492 partition: p.partition.clone(),
1493 record_count: p.record_count,
1494 location: loc.clone(),
1495 meta: &p.meta,
1496 epoch: p.epoch,
1497 })
1498 .collect();
1499
1500 let commit_started = timing.then(Instant::now);
1501 let _ = blob.take_media_op_stats();
1504 let mut commit_error = None;
1505 let committed = if work
1506 .batch
1507 .iter()
1508 .any(|pending| pending.expected_head.is_some())
1509 {
1510 let heads: Vec<_> = work
1511 .batch
1512 .iter()
1513 .map(|pending| pending.expected_head)
1514 .collect();
1515 sequencer
1516 .commit_conditionally(&commit_batches, &heads)
1517 .map(CommitResults::Conditional)
1518 } else {
1519 sequencer.commit(&commit_batches).map(CommitResults::Legacy)
1520 }
1521 .and_then(|outcomes| {
1522 if outcomes.len() != commit_batches.len() {
1523 return Err(ObjectLogError::Sequencer(
1524 "wrong commit outcome count".into(),
1525 ));
1526 }
1527 Ok(outcomes)
1528 });
1529 match committed {
1530 Ok(outcomes) => {
1531 if let Some(commit_started) = commit_started {
1532 eprintln!(
1533 "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1534 work.bytes,
1535 commit_batches.len(),
1536 put_elapsed.as_millis(),
1537 commit_started.elapsed().as_millis(),
1538 work.data_objects
1539 );
1540 }
1541 if let Some(s) = blob.take_media_op_stats() {
1542 media_ops = media_ops.saturating_add(s.media_ops);
1543 }
1544 for ((outcome, (_, tx)), pending) in outcomes
1545 .into_iter()
1546 .zip(work.responders.iter_mut())
1547 .zip(work.batch.iter())
1548 {
1549 if let Some(tx) = tx.take() {
1550 let outcome = match outcome {
1551 ConditionalCommitOutcome::Committed(outcome) => outcome,
1552 ConditionalCommitOutcome::HeadRejected { expected, current } => {
1553 let _ =
1554 tx.send(Err(ObjectLogError::HeadRejected { expected, current }));
1555 continue;
1556 }
1557 ConditionalCommitOutcome::Unknown(error) => {
1558 let _ = tx.send(Err(ObjectLogError::StorageUnavailable(format!(
1559 "conditional publication outcome unknown: {error}"
1560 ))));
1561 continue;
1562 }
1563 };
1564 let (base, last) = match outcome {
1565 CommitOutcome::Assigned {
1566 base_offset,
1567 record_count,
1568 } => (
1569 Some(base_offset),
1570 Some(base_offset + record_count as i64 - 1),
1571 ),
1572 CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1573 CommitOutcome::Rejected { reason } => {
1574 let error = sequencer.rejection_error(
1576 &pending.partition,
1577 pending.epoch,
1578 reason,
1579 );
1580 let _ = tx.send(Err(error));
1581 continue;
1582 }
1583 };
1584 let _ = tx.send(Ok(AppendOutcome {
1585 base_offset: base,
1586 last_offset: last,
1587 durable: true,
1588 sequenced: true,
1589 }));
1590 }
1591 }
1592 }
1593 Err(e) => {
1594 if let Some(s) = blob.take_media_op_stats() {
1595 media_ops = media_ops.saturating_add(s.media_ops);
1596 }
1597 send_storage_error(&mut work.responders, e.clone());
1598 commit_error = Some(e);
1599 }
1600 }
1601
1602 {
1603 let mut budget = shared.budget.lock().expect("poisoned");
1604 budget.consume_after_flush(media_ops, Instant::now());
1605 }
1606 {
1607 let mut q = shared.queue.lock().expect("poisoned");
1608 if let Some(error) = commit_error {
1609 q.first_failed.get_or_insert((first_seq, error));
1610 }
1611 if work.max_seq >= q.completed_through {
1612 q.completed_through = work.max_seq;
1613 }
1614 notify_flush_waiters(&mut q);
1615 }
1616 shared.cv.notify_all();
1617
1618 release_bytes
1619}
1620
1621#[cfg(test)]
1622mod flush_barrier_tests {
1623 use super::*;
1624
1625 #[test]
1626 fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1627 let error = ObjectLogError::Sequencer("manifest failed".into());
1628 let mut queue = Queue::<()> {
1629 items: VecDeque::new(),
1630 bytes: 0,
1631 bytes_in_use: 1,
1632 shutdown: false,
1633 next_seq: 4,
1634 completed_through: 2,
1635 first_failed: Some((2, error.clone())),
1636 force_flush: true,
1637 flush_waiters: Vec::new(),
1638 last_enqueue: None,
1639 oldest_enqueue: None,
1640 };
1641 let mut receivers = Vec::new();
1642 for barrier in [1, 2, 3] {
1643 let (tx, rx) = oneshot::channel();
1644 queue.flush_waiters.push((barrier, tx));
1645 receivers.push(rx);
1646 }
1647 notify_flush_waiters(&mut queue);
1648 assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1649 assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1650 assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1651 assert!(queue.flush_waiters.is_empty());
1652 }
1653}
1654
1655#[cfg(test)]
1656mod ready_commit_group_tests {
1657 use super::*;
1658 use crate::{ManifestSequencer, MemoryBlobStore};
1659 use async_trait::async_trait;
1660 use std::ops::Range;
1661
1662 type WorkFixture = (
1663 FlushWork<()>,
1664 String,
1665 Vec<Bytes>,
1666 oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1667 );
1668 fn work(n: u64, durability: Durability) -> WorkFixture {
1669 let (tx, rx) = oneshot::channel();
1670 let (mut work, key, chunks) = prepare_flush_work(
1671 "data/",
1672 "",
1673 n,
1674 vec![Pending {
1675 partition: PartitionKey("shared".into()),
1676 record_count: 1,
1677 payload: Bytes::from(format!("body-{n}")),
1678 meta: (),
1679 epoch: 0,
1680 durability,
1681 responder: Some(tx),
1682 seq: n,
1683 owner: None,
1684 expected_head: None,
1685 }],
1686 );
1687 work.put_result = Some(Ok(Duration::from_millis(n)));
1688 (work, key, chunks, rx)
1689 }
1690
1691 #[test]
1692 fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1693 let mut pending = (1..=5)
1694 .map(|n| work(n, Durability::Sequenced).0)
1695 .collect::<VecDeque<_>>();
1696 let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1697 let mut released = 0;
1698 for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1699 let group = take_ready_commit_group(&mut pending, 2).unwrap();
1700 assert_eq!(
1701 group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1702 expected
1703 );
1704 assert_eq!(group.data_objects, expected.len());
1705 assert_eq!(group.responders.len(), expected.len());
1706 assert_eq!(group.max_seq, *expected.last().unwrap());
1707 for (location, n) in group.locations.iter().zip(&expected) {
1708 assert_eq!(location.object_id, format!("data/{n:020}"));
1709 assert_eq!(location.byte_start, 0);
1710 assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1711 }
1712 released += group.bytes;
1713 }
1714 assert!(pending.is_empty());
1715 assert_eq!(released, original_bytes);
1716
1717 let mut first = work(1, Durability::Sequenced).0;
1718 first.put_result = None;
1719 let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1720 assert!(take_ready_commit_group(&mut pending, 8).is_none());
1721 assert_eq!(
1722 pending.len(),
1723 2,
1724 "a later ready PUT cannot pass an unfinished head"
1725 );
1726 pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1727 let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1728 assert_eq!(failed.data_objects, 1);
1729 assert!(failed.put_result.unwrap().is_err());
1730 assert_eq!(pending[0].max_seq, 2);
1731
1732 let mut failed = work(2, Durability::Sequenced).0;
1733 failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1734 let mut pending = VecDeque::from([
1735 work(1, Durability::Sequenced).0,
1736 failed,
1737 work(3, Durability::Sequenced).0,
1738 ]);
1739 assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1740 assert_eq!(
1741 pending.len(),
1742 2,
1743 "a success group cannot cross a failed PUT"
1744 );
1745 let mut pending = VecDeque::from([
1746 work(1, Durability::Sequenced).0,
1747 work(2, Durability::Sequenced).0,
1748 ]);
1749 assert_eq!(
1750 take_ready_commit_group(&mut pending, 1)
1751 .unwrap()
1752 .data_objects,
1753 1
1754 );
1755 assert_eq!(
1756 pending.len(),
1757 1,
1758 "one-object sequencers retain their call boundary"
1759 );
1760 }
1761
1762 struct GatedManifestStore {
1763 inner: MemoryBlobStore,
1764 entered: std::sync::mpsc::SyncSender<()>,
1765 release: Mutex<Option<oneshot::Receiver<()>>>,
1766 fail: bool,
1767 panic_on_commit: bool,
1768 }
1769
1770 #[async_trait]
1771 impl BlobStore for GatedManifestStore {
1772 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1773 if key.starts_with("manifest/") {
1774 let release = self.release.lock().unwrap().take();
1775 if let Some(release) = release {
1776 self.entered.send(()).unwrap();
1777 release.await.unwrap();
1778 assert!(!self.panic_on_commit, "injected commit panic");
1779 if self.fail {
1780 return Err(ObjectLogError::StorageUnavailable(
1781 "manifest failure".into(),
1782 ));
1783 }
1784 }
1785 }
1786 self.inner.put(key, value).await
1787 }
1788 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1789 self.inner.get(key).await
1790 }
1791 async fn get_range(
1792 &self,
1793 key: &str,
1794 range: Range<u64>,
1795 ) -> Result<Option<Bytes>, ObjectLogError> {
1796 self.inner.get_range(key, range).await
1797 }
1798 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1799 self.inner.list(prefix).await
1800 }
1801 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1802 self.inner.delete(key).await
1803 }
1804 fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1805 self.inner.take_media_op_stats()
1806 }
1807 }
1808
1809 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1810 async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1811 for fail in [false, true] {
1812 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1813 let (release_tx, release_rx) = oneshot::channel();
1814 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1815 inner: MemoryBlobStore::new(),
1816 entered: entered_tx,
1817 release: Mutex::new(Some(release_rx)),
1818 fail,
1819 panic_on_commit: false,
1820 });
1821 let sequencer = Arc::new(
1822 ManifestSequencer::open(blob.clone(), "manifest/")
1823 .await
1824 .unwrap(),
1825 );
1826 assert!(sequencer.supports_multi_object_commit());
1827 let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1828 let (mut second, second_key, second_chunks, mut durable_rx) =
1829 work(2, Durability::Durable);
1830 second.batch[0].partition = PartitionKey("other".into());
1831 let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1832 blob.put_chunks(&first_key, first_chunks).await.unwrap();
1833 blob.put_chunks(&second_key, second_chunks).await.unwrap();
1834 blob.put_chunks(&third_key, third_chunks).await.unwrap();
1835 let mut pending = VecDeque::from([first, second, third]);
1836 let group = take_ready_commit_group(&mut pending, 8).unwrap();
1837 let bytes = group.bytes;
1838 assert_eq!(group.data_objects, 3);
1839 let (flush_tx, mut flush_rx) = oneshot::channel();
1840 let config = FlushConfig {
1841 budget: BudgetConfig {
1842 enabled: false,
1843 ..BudgetConfig::default()
1844 },
1845 ..FlushConfig::default()
1846 };
1847 let shared: Arc<Shared<()>> = Arc::new(Shared {
1848 queue: Mutex::new(Queue {
1849 items: VecDeque::new(),
1850 bytes: 0,
1851 bytes_in_use: bytes,
1852 shutdown: false,
1853 next_seq: 4,
1854 completed_through: 0,
1855 first_failed: None,
1856 force_flush: true,
1857 flush_waiters: vec![(3, flush_tx)],
1858 last_enqueue: None,
1859 oldest_enqueue: None,
1860 }),
1861 cv: Condvar::new(),
1862 max_buffered_bytes: bytes,
1863 budget: Mutex::new(BudgetRuntime::new(config.budget)),
1864 flush_config: config,
1865 });
1866 let worker = {
1867 let shared = shared.clone();
1868 let blob = blob.clone();
1869 let sequencer = sequencer.clone();
1870 std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1871 };
1872 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1873 assert!(matches!(
1874 sequenced_rx.try_recv(),
1875 Err(oneshot::error::TryRecvError::Empty)
1876 ));
1877 assert!(matches!(
1878 third_rx.try_recv(),
1879 Err(oneshot::error::TryRecvError::Empty)
1880 ));
1881 assert!(matches!(
1882 flush_rx.try_recv(),
1883 Err(oneshot::error::TryRecvError::Empty)
1884 ));
1885 let durable = durable_rx.try_recv().unwrap().unwrap();
1886 assert!(durable.durable && !durable.sequenced);
1887 release_tx.send(()).unwrap();
1888 assert_eq!(worker.join().unwrap(), bytes);
1889 if fail {
1890 assert!(sequenced_rx.await.unwrap().is_err());
1891 assert!(third_rx.await.unwrap().is_err());
1892 assert!(flush_rx.await.unwrap().is_err());
1893 assert_eq!(
1894 sequencer
1895 .high_watermark(&PartitionKey("shared".into()))
1896 .unwrap(),
1897 0
1898 );
1899 assert!(blob.list("manifest/").await.unwrap().is_empty());
1900 } else {
1901 assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1902 assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1903 flush_rx.await.unwrap().unwrap();
1904 assert_eq!(blob.list("manifest/").await.unwrap().len(), 3);
1905 }
1906 assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1907 let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1908 .await
1909 .unwrap();
1910 for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1911 let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1912 assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1913 for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1914 assert_eq!(entry.base_offset, offset as i64);
1915 let loc = &entry.location;
1916 assert_eq!(loc.object_id, format!("data/{object:020}"));
1917 let body = blob
1918 .get_range(
1919 &loc.object_id,
1920 loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1921 )
1922 .await
1923 .unwrap()
1924 .unwrap();
1925 assert_eq!(body, format!("body-{object}"));
1926 }
1927 }
1928 }
1929 }
1930
1931 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1932 async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1933 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1934 let (release_tx, release_rx) = oneshot::channel();
1935 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1936 inner: MemoryBlobStore::new(),
1937 entered: entered_tx,
1938 release: Mutex::new(Some(release_rx)),
1939 fail: false,
1940 panic_on_commit: false,
1941 });
1942 let sequencer = Arc::new(
1943 ManifestSequencer::open(blob.clone(), "manifest/")
1944 .await
1945 .unwrap(),
1946 );
1947 let config = FlushConfig {
1948 max_batches: 1,
1949 max_inflight_flushes: 4,
1950 linger: Duration::ZERO,
1951 budget: BudgetConfig {
1952 enabled: false,
1953 ..BudgetConfig::default()
1954 },
1955 ..FlushConfig::default()
1956 };
1957 let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1958 let first = {
1959 let engine = engine.clone();
1960 tokio::spawn(async move {
1961 engine
1962 .produce(
1963 PartitionKey("p".into()),
1964 Bytes::from_static(b"first"),
1965 1,
1966 (),
1967 Durability::Sequenced,
1968 )
1969 .await
1970 })
1971 };
1972 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1973 for n in 1..4 {
1974 engine
1975 .produce(
1976 PartitionKey("p".into()),
1977 Bytes::from(format!("later-{n}")),
1978 1,
1979 (),
1980 Durability::Buffered,
1981 )
1982 .await
1983 .unwrap();
1984 }
1985 let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1986 loop {
1987 if blob.list("data/").await.unwrap().len() == 4 {
1988 break;
1989 }
1990 tokio::time::sleep(Duration::from_millis(1)).await;
1991 }
1992 })
1993 .await
1994 .is_ok();
1995 let early_ack = first.is_finished();
1996 release_tx.send(()).unwrap();
1998 first.await.unwrap().unwrap();
1999 engine.flush().await.unwrap();
2000 let rows = engine
2001 .fetch(&PartitionKey("p".into()), 0, 1024)
2002 .await
2003 .unwrap();
2004 assert_eq!(
2005 rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
2006 vec![0, 1, 2, 3]
2007 );
2008 drop(engine);
2009 assert!(
2010 !early_ack,
2011 "sequenced acknowledgement preceded manifest durability"
2012 );
2013 assert!(
2014 upload_progress,
2015 "blocked manifest prevented independent data uploads"
2016 );
2017 }
2018
2019 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2020 async fn commit_completion_releases_admission_bytes_without_more_uploads() {
2021 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
2022 let sequencer = Arc::new(
2023 ManifestSequencer::open(blob.clone(), "manifest/")
2024 .await
2025 .unwrap(),
2026 );
2027 let config = FlushConfig {
2028 max_bytes: 4,
2029 max_buffered_bytes: 4,
2030 max_batches: 1,
2031 max_inflight_flushes: 4,
2032 linger: Duration::ZERO,
2033 budget: BudgetConfig {
2034 enabled: false,
2035 ..BudgetConfig::default()
2036 },
2037 };
2038 let engine = LogEngine::new(blob, sequencer, config, "data/");
2039 engine
2040 .produce(
2041 PartitionKey("p".into()),
2042 Bytes::from_static(b"full"),
2043 1,
2044 (),
2045 Durability::Sequenced,
2046 )
2047 .await
2048 .unwrap();
2049 tokio::time::timeout(Duration::from_secs(5), async {
2050 while engine.buffer_stats().bytes_in_use != 0 {
2051 tokio::time::sleep(Duration::from_millis(1)).await;
2052 }
2053 })
2054 .await
2055 .expect("completed committer stranded admission bytes");
2056 engine
2057 .produce(
2058 PartitionKey("p".into()),
2059 Bytes::from_static(b"next"),
2060 1,
2061 (),
2062 Durability::Sequenced,
2063 )
2064 .await
2065 .unwrap();
2066 engine.flush().await.unwrap();
2067 assert_eq!(
2068 engine
2069 .fetch(&PartitionKey("p".into()), 0, 100)
2070 .await
2071 .unwrap()
2072 .len(),
2073 2
2074 );
2075 }
2076
2077 struct FailureInjectingStore {
2078 inner: Arc<dyn BlobStore>,
2079 second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
2080 second_upload_completed: std::sync::atomic::AtomicBool,
2081 fail_data_listing: std::sync::atomic::AtomicBool,
2082 }
2083
2084 impl FailureInjectingStore {
2085 fn new(inner: Arc<dyn BlobStore>) -> Self {
2086 Self {
2087 inner,
2088 second_upload_gate: Mutex::new(None),
2089 second_upload_completed: std::sync::atomic::AtomicBool::new(false),
2090 fail_data_listing: std::sync::atomic::AtomicBool::new(false),
2091 }
2092 }
2093 }
2094
2095 #[async_trait]
2096 impl BlobStore for FailureInjectingStore {
2097 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
2098 let gate = if key == "data/00000000000000000002" {
2099 self.second_upload_gate.lock().unwrap().take()
2100 } else {
2101 None
2102 };
2103 if let Some((entered, release)) = gate {
2104 entered.send(()).unwrap();
2105 release.await.unwrap();
2106 let result = self.inner.put(key, value).await;
2107 if result.is_ok() {
2108 self.second_upload_completed
2109 .store(true, std::sync::atomic::Ordering::SeqCst);
2110 }
2111 return result;
2112 }
2113 self.inner.put(key, value).await
2114 }
2115 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
2116 self.inner.get(key).await
2117 }
2118 async fn get_range(
2119 &self,
2120 key: &str,
2121 range: Range<u64>,
2122 ) -> Result<Option<Bytes>, ObjectLogError> {
2123 self.inner.get_range(key, range).await
2124 }
2125 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
2126 if prefix == "data/"
2127 && self
2128 .fail_data_listing
2129 .load(std::sync::atomic::Ordering::SeqCst)
2130 {
2131 return Err(ObjectLogError::StorageUnavailable(
2132 "injected object listing failure".into(),
2133 ));
2134 }
2135 self.inner.list(prefix).await
2136 }
2137 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
2138 self.inner.delete(key).await
2139 }
2140 }
2141
2142 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2143 async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
2144 let inner = Arc::new(MemoryBlobStore::new());
2145 let key = "data/00000000000000000001";
2146 let original = Bytes::from_static(b"existing immutable object");
2147 inner.put(key, original.clone()).await.unwrap();
2148 let store = Arc::new(FailureInjectingStore::new(inner.clone()));
2149 store
2150 .fail_data_listing
2151 .store(true, std::sync::atomic::Ordering::SeqCst);
2152 let config = FlushConfig {
2153 max_batches: 1,
2154 max_inflight_flushes: 4,
2155 linger: Duration::ZERO,
2156 budget: BudgetConfig {
2157 enabled: false,
2158 ..BudgetConfig::default()
2159 },
2160 ..FlushConfig::default()
2161 };
2162 let engine = LogEngine::new(
2163 store,
2164 Arc::new(crate::InMemorySequencer::new()),
2165 config,
2166 "data/",
2167 );
2168 let result = tokio::time::timeout(
2169 Duration::from_secs(5),
2170 engine.produce(
2171 PartitionKey("p".into()),
2172 Bytes::from_static(b"must not overwrite"),
2173 1,
2174 (),
2175 Durability::Sequenced,
2176 ),
2177 )
2178 .await;
2179 let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
2180 tokio::task::spawn_blocking(move || drop(engine))
2181 .await
2182 .unwrap();
2183 let retained = inner.get(key).await.unwrap();
2184 assert_eq!(
2185 retained,
2186 Some(original),
2187 "failed listing must preserve sealed bytes"
2188 );
2189 assert!(
2190 result.unwrap().is_err(),
2191 "failed prefix recovery must reject admission"
2192 );
2193 assert!(
2194 flushed.unwrap().is_err(),
2195 "flush must report failed initialization"
2196 );
2197 assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
2198 }
2199
2200 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2201 async fn engines_commit_and_close_independently() {
2202 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2203 let (release_tx, release_rx) = oneshot::channel();
2204 let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2205 inner: MemoryBlobStore::new(),
2206 entered: entered_tx,
2207 release: Mutex::new(Some(release_rx)),
2208 fail: false,
2209 panic_on_commit: false,
2210 })));
2211 let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
2212 let config = FlushConfig {
2213 max_batches: 1,
2214 max_inflight_flushes: 4,
2215 linger: Duration::ZERO,
2216 budget: BudgetConfig {
2217 enabled: false,
2218 ..BudgetConfig::default()
2219 },
2220 ..FlushConfig::default()
2221 };
2222 let a_seq = Arc::new(
2223 ManifestSequencer::open(a_blob.clone(), "manifest/")
2224 .await
2225 .unwrap(),
2226 );
2227 let b_seq = Arc::new(
2228 ManifestSequencer::open(b_blob.clone(), "manifest/")
2229 .await
2230 .unwrap(),
2231 );
2232 let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
2233 a.produce(
2234 PartitionKey("p".into()),
2235 Bytes::from_static(b"a-first"),
2236 1,
2237 (),
2238 Durability::Buffered,
2239 )
2240 .await
2241 .unwrap();
2242 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2243 let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
2244 let b_result = tokio::time::timeout(
2245 Duration::from_secs(5),
2246 b.produce(
2247 PartitionKey("p".into()),
2248 Bytes::from_static(b"b-first"),
2249 1,
2250 (),
2251 Durability::Sequenced,
2252 ),
2253 )
2254 .await;
2255 let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
2256 let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
2257 release_tx.send(()).unwrap();
2259 if b_closed.is_err() {
2260 closing_b.await.unwrap();
2261 }
2262 assert!(
2263 b_closed.is_ok(),
2264 "closing B waited for A's blocked manifest"
2265 );
2266 assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
2267 a.flush().await.unwrap();
2268 assert_eq!(
2269 a.produce(
2270 PartitionKey("p".into()),
2271 Bytes::from_static(b"a-second"),
2272 1,
2273 (),
2274 Durability::Sequenced
2275 )
2276 .await
2277 .unwrap()
2278 .base_offset,
2279 Some(1)
2280 );
2281 let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
2282 assert_eq!(
2283 rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
2284 vec![
2285 Bytes::from_static(b"a-first"),
2286 Bytes::from_static(b"a-second")
2287 ]
2288 );
2289 tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
2290 }
2291
2292 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2293 async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
2294 let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
2295 let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
2296 let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
2297 let (upload_release_tx, upload_release_rx) = oneshot::channel();
2298 let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2299 inner: MemoryBlobStore::new(),
2300 entered: manifest_entered_tx,
2301 release: Mutex::new(Some(manifest_release_rx)),
2302 fail: false,
2303 panic_on_commit: true,
2304 })));
2305 *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
2306 let config = FlushConfig {
2307 max_batches: 1,
2308 max_inflight_flushes: 4,
2309 linger: Duration::ZERO,
2310 budget: BudgetConfig {
2311 enabled: false,
2312 ..BudgetConfig::default()
2313 },
2314 ..FlushConfig::default()
2315 };
2316 let sequencer = Arc::new(
2317 ManifestSequencer::open(store.clone(), "manifest/")
2318 .await
2319 .unwrap(),
2320 );
2321 let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
2322 engine
2323 .produce(
2324 PartitionKey("p".into()),
2325 Bytes::from_static(b"first"),
2326 1,
2327 (),
2328 Durability::Buffered,
2329 )
2330 .await
2331 .unwrap();
2332 manifest_entered_rx
2333 .recv_timeout(Duration::from_secs(5))
2334 .unwrap();
2335 engine
2336 .produce(
2337 PartitionKey("p".into()),
2338 Bytes::from_static(b"second"),
2339 1,
2340 (),
2341 Durability::Buffered,
2342 )
2343 .await
2344 .unwrap();
2345 upload_entered_rx
2346 .recv_timeout(Duration::from_secs(5))
2347 .unwrap();
2348 manifest_release_tx.send(()).unwrap();
2349 tokio::time::timeout(Duration::from_secs(5), async {
2350 while !engine.shared.queue.lock().unwrap().shutdown {
2351 tokio::time::sleep(Duration::from_millis(1)).await;
2352 }
2353 })
2354 .await
2355 .unwrap();
2356 let flush_failed = engine.flush().await.is_err();
2357 let upload_was_owned = upload_release_tx.send(()).is_ok();
2360 tokio::time::timeout(
2361 Duration::from_secs(5),
2362 tokio::task::spawn_blocking(move || drop(engine)),
2363 )
2364 .await
2365 .unwrap()
2366 .unwrap();
2367 assert!(flush_failed);
2368 assert!(upload_was_owned);
2369 assert!(
2370 store
2371 .second_upload_completed
2372 .load(std::sync::atomic::Ordering::SeqCst)
2373 );
2374 assert_eq!(store.list("data/").await.unwrap().len(), 2);
2375 let sequencer = Arc::new(
2376 ManifestSequencer::open(store.clone(), "manifest/")
2377 .await
2378 .unwrap(),
2379 );
2380 let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2381 assert!(
2382 reopened
2383 .fetch(&PartitionKey("p".into()), 0, 100)
2384 .await
2385 .unwrap()
2386 .is_empty()
2387 );
2388 assert_eq!(
2389 reopened
2390 .produce(
2391 PartitionKey("p".into()),
2392 Bytes::from_static(b"after-reopen"),
2393 1,
2394 (),
2395 Durability::Sequenced
2396 )
2397 .await
2398 .unwrap()
2399 .base_offset,
2400 Some(0)
2401 );
2402 let rows = reopened
2403 .fetch(&PartitionKey("p".into()), 0, 100)
2404 .await
2405 .unwrap();
2406 assert_eq!(rows.len(), 1);
2407 assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2408 assert_eq!(store.list("data/").await.unwrap().len(), 3);
2409 tokio::task::spawn_blocking(move || drop(reopened))
2410 .await
2411 .unwrap();
2412 }
2413
2414 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2415 async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2416 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2417 let (release_tx, release_rx) = oneshot::channel();
2418 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2419 inner: MemoryBlobStore::new(),
2420 entered: entered_tx,
2421 release: Mutex::new(Some(release_rx)),
2422 fail: false,
2423 panic_on_commit: false,
2424 });
2425 let sequencer = Arc::new(
2426 ManifestSequencer::open(blob.clone(), "manifest/")
2427 .await
2428 .unwrap(),
2429 );
2430 let config = FlushConfig {
2431 max_batches: 1,
2432 max_inflight_flushes: 4,
2433 linger: Duration::ZERO,
2434 budget: BudgetConfig {
2435 enabled: false,
2436 ..BudgetConfig::default()
2437 },
2438 ..FlushConfig::default()
2439 };
2440 let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2441 engine
2442 .produce(
2443 PartitionKey("p".into()),
2444 Bytes::from_static(b"first"),
2445 1,
2446 (),
2447 Durability::Buffered,
2448 )
2449 .await
2450 .unwrap();
2451 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2452 engine
2453 .produce(
2454 PartitionKey("p".into()),
2455 Bytes::from_static(b"second"),
2456 1,
2457 (),
2458 Durability::Buffered,
2459 )
2460 .await
2461 .unwrap();
2462 let shared = engine.shared.clone();
2463 let dropping = tokio::task::spawn_blocking(move || drop(engine));
2464 tokio::time::timeout(Duration::from_secs(5), async {
2465 while !shared.queue.lock().unwrap().shutdown {
2466 tokio::time::sleep(Duration::from_millis(1)).await;
2467 }
2468 })
2469 .await
2470 .unwrap();
2471 let premature = dropping.is_finished();
2472 release_tx.send(()).unwrap();
2473 tokio::time::timeout(Duration::from_secs(5), dropping)
2474 .await
2475 .unwrap()
2476 .unwrap();
2477 assert!(!premature, "shutdown returned before manifest durability");
2478 let reopened = Arc::new(
2479 ManifestSequencer::open(blob.clone(), "manifest/")
2480 .await
2481 .unwrap(),
2482 );
2483 let engine = LogEngine::new(blob, reopened, config, "data/");
2484 let rows = engine
2485 .fetch(&PartitionKey("p".into()), 0, 100)
2486 .await
2487 .unwrap();
2488 assert_eq!(
2489 rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2490 vec![b"first".as_slice(), b"second".as_slice()]
2491 );
2492 }
2493
2494 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2495 async fn panicked_committer_closes_admission_and_fails_flush() {
2496 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2497 let (release_tx, release_rx) = oneshot::channel();
2498 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2499 inner: MemoryBlobStore::new(),
2500 entered: entered_tx,
2501 release: Mutex::new(Some(release_rx)),
2502 fail: false,
2503 panic_on_commit: true,
2504 });
2505 let sequencer = Arc::new(
2506 ManifestSequencer::open(blob.clone(), "manifest/")
2507 .await
2508 .unwrap(),
2509 );
2510 let config = FlushConfig {
2511 max_batches: 1,
2512 max_inflight_flushes: 4,
2513 linger: Duration::ZERO,
2514 budget: BudgetConfig {
2515 enabled: false,
2516 ..BudgetConfig::default()
2517 },
2518 ..FlushConfig::default()
2519 };
2520 let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2521 let first = {
2522 let engine = engine.clone();
2523 tokio::spawn(async move {
2524 engine
2525 .produce(
2526 PartitionKey("p".into()),
2527 Bytes::from_static(b"first"),
2528 1,
2529 (),
2530 Durability::Sequenced,
2531 )
2532 .await
2533 })
2534 };
2535 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2536 engine
2537 .produce(
2538 PartitionKey("p".into()),
2539 Bytes::from_static(b"later"),
2540 1,
2541 (),
2542 Durability::Buffered,
2543 )
2544 .await
2545 .unwrap();
2546 let flushing = {
2547 let engine = engine.clone();
2548 tokio::spawn(async move { engine.flush().await })
2549 };
2550 release_tx.send(()).unwrap();
2551 assert!(
2552 tokio::time::timeout(Duration::from_secs(5), first)
2553 .await
2554 .unwrap()
2555 .unwrap()
2556 .is_err()
2557 );
2558 assert!(
2559 tokio::time::timeout(Duration::from_secs(5), flushing)
2560 .await
2561 .unwrap()
2562 .unwrap()
2563 .is_err()
2564 );
2565 tokio::time::timeout(Duration::from_secs(5), async {
2566 while !engine.shared.queue.lock().unwrap().shutdown {
2567 tokio::time::sleep(Duration::from_millis(1)).await;
2568 }
2569 })
2570 .await
2571 .unwrap();
2572 assert!(
2573 engine
2574 .produce(
2575 PartitionKey("p".into()),
2576 Bytes::from_static(b"rejected"),
2577 1,
2578 (),
2579 Durability::Buffered
2580 )
2581 .await
2582 .is_err()
2583 );
2584 assert!(engine.flush().await.is_err());
2585 }
2586 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2587 async fn committed_reads_continue_during_new_manifest_publication() {
2588 for fail in [false, true] {
2589 let memory = MemoryBlobStore::new();
2590 let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2591 let seed_sequencer = Arc::new(
2592 ManifestSequencer::open(seed_blob.clone(), "manifest/")
2593 .await
2594 .unwrap(),
2595 );
2596 let config = FlushConfig {
2597 max_batches: 1,
2598 max_inflight_flushes: 4,
2599 linger: Duration::ZERO,
2600 budget: BudgetConfig {
2601 enabled: false,
2602 ..BudgetConfig::default()
2603 },
2604 ..FlushConfig::default()
2605 };
2606 let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2607 seed.produce(
2608 PartitionKey("p".into()),
2609 Bytes::from_static(b"committed"),
2610 1,
2611 (),
2612 Durability::Sequenced,
2613 )
2614 .await
2615 .unwrap();
2616 drop(seed);
2617 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2618 let (release_tx, release_rx) = oneshot::channel();
2619 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2620 inner: memory,
2621 entered: entered_tx,
2622 release: Mutex::new(Some(release_rx)),
2623 fail,
2624 panic_on_commit: false,
2625 });
2626 let sequencer = Arc::new(
2627 ManifestSequencer::open(blob.clone(), "manifest/")
2628 .await
2629 .unwrap(),
2630 );
2631 let engine = Arc::new(LogEngine::new(
2632 blob.clone(),
2633 sequencer.clone(),
2634 config,
2635 "data/",
2636 ));
2637 let producing = {
2638 let engine = engine.clone();
2639 tokio::spawn(async move {
2640 engine
2641 .produce(
2642 PartitionKey("p".into()),
2643 Bytes::from_static(b"new"),
2644 1,
2645 (),
2646 Durability::Sequenced,
2647 )
2648 .await
2649 })
2650 };
2651 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2652 let mut reading = tokio::task::spawn_blocking(move || {
2653 let p = PartitionKey("p".into());
2654 (
2655 sequencer.high_watermark(&p).unwrap(),
2656 sequencer.lookup(&p, 0).unwrap(),
2657 sequencer.snapshot(),
2658 )
2659 });
2660 let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2661 let early_ack = producing.is_finished();
2662 release_tx.send(()).unwrap();
2664 let outcome = producing.await.unwrap();
2665 assert_eq!(outcome.is_err(), fail);
2666 let reads_progressed = observed.is_ok();
2667 let (watermark, entries, snapshot) = match observed {
2668 Ok(result) => result.unwrap(),
2669 Err(_) => reading.await.unwrap(),
2670 };
2671 assert!(!early_ack);
2672 assert!(
2673 reads_progressed,
2674 "durable manifest publication blocked committed index reads"
2675 );
2676 assert_eq!(watermark, 1);
2677 assert_eq!(entries.len(), 1);
2678 assert_eq!(snapshot.manifest_count, 1);
2679 let rows = engine
2680 .fetch(&PartitionKey("p".into()), 0, 100)
2681 .await
2682 .unwrap();
2683 assert_eq!(rows.len(), if fail { 1 } else { 2 });
2684 assert_eq!(rows[0].payload.as_ref(), b"committed");
2685 drop(engine);
2686 let reopened = Arc::new(
2687 ManifestSequencer::open(blob.clone(), "manifest/")
2688 .await
2689 .unwrap(),
2690 );
2691 let engine = LogEngine::new(blob, reopened, config, "data/");
2692 assert_eq!(
2693 engine
2694 .fetch(&PartitionKey("p".into()), 0, 100)
2695 .await
2696 .unwrap()
2697 .len(),
2698 if fail { 1 } else { 2 }
2699 );
2700 }
2701 }
2702 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2703 async fn manifest_mutations_remain_serialized_while_readers_progress() {
2704 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2705 let (release_tx, release_rx) = oneshot::channel();
2706 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2707 inner: MemoryBlobStore::new(),
2708 entered: entered_tx,
2709 release: Mutex::new(Some(release_rx)),
2710 fail: false,
2711 panic_on_commit: false,
2712 });
2713 let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2714 let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2715 tokio::task::spawn_blocking(move || {
2716 sequencer.commit(&[CommitBatch {
2717 partition: PartitionKey("p".into()),
2718 record_count: 1,
2719 location: BatchLocation {
2720 object_id: key.into(),
2721 byte_start: 0,
2722 byte_len: 1,
2723 },
2724 meta: &(),
2725 epoch: 0,
2726 }])
2727 })
2728 };
2729 let first = commit(sequencer.clone(), "data/first");
2730 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2731 let second = commit(sequencer.clone(), "data/second");
2732 let truncating = {
2733 let sequencer = sequencer.clone();
2734 tokio::task::spawn_blocking(move || {
2735 sequencer.truncate_before(&PartitionKey("p".into()), 1)
2736 })
2737 };
2738 let reading = {
2739 let sequencer = sequencer.clone();
2740 tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2741 };
2742 let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2743 release_tx.send(()).unwrap();
2744 assert_eq!(
2745 observed.expect("committed reads blocked").unwrap().unwrap(),
2746 0
2747 );
2748 let first = tokio::time::timeout(Duration::from_secs(5), first)
2749 .await
2750 .unwrap()
2751 .unwrap()
2752 .unwrap();
2753 let second = tokio::time::timeout(Duration::from_secs(5), second)
2754 .await
2755 .unwrap()
2756 .unwrap()
2757 .unwrap();
2758 let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2759 .await
2760 .unwrap()
2761 .unwrap()
2762 .unwrap();
2763 assert_eq!(
2764 first,
2765 vec![CommitOutcome::Assigned {
2766 base_offset: 0,
2767 record_count: 1
2768 }]
2769 );
2770 assert_eq!(
2771 second,
2772 vec![CommitOutcome::Assigned {
2773 base_offset: 1,
2774 record_count: 1
2775 }]
2776 );
2777 assert_eq!(dropped, vec!["data/first".to_string()]);
2778 let p = PartitionKey("p".into());
2779 assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2780 assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2781 let entries = sequencer.lookup(&p, 0).unwrap();
2782 assert_eq!(entries.len(), 1);
2783 assert_eq!(entries[0].location.object_id, "data/second");
2784 assert_eq!(sequencer.snapshot().manifest_count, 2);
2785 }
2786}
2787
2788#[cfg(test)]
2789mod idle_batch_wait_tests {
2790 use super::*;
2791 use std::sync::mpsc;
2792
2793 fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2794 Arc::new(Shared {
2795 queue: Mutex::new(Queue {
2796 items: VecDeque::new(),
2797 bytes: 0,
2798 bytes_in_use: 0,
2799 shutdown: false,
2800 next_seq: 1,
2801 completed_through: 0,
2802 first_failed: None,
2803 force_flush: false,
2804 flush_waiters: Vec::new(),
2805 last_enqueue: None,
2806 oldest_enqueue: None,
2807 }),
2808 cv: Condvar::new(),
2809 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2810 budget: Mutex::new(BudgetRuntime::new(config.budget)),
2811 flush_config: config,
2812 })
2813 }
2814
2815 #[test]
2816 fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2817 for linger in [Duration::ZERO, Duration::from_millis(1)] {
2818 for shutdown in [false, true] {
2819 let config = FlushConfig {
2820 linger,
2821 ..FlushConfig::default()
2822 };
2823 let shared = empty_shared(config);
2824 let (started_tx, started_rx) = mpsc::channel();
2825 let (result_tx, result_rx) = mpsc::channel();
2826 let worker_shared = shared.clone();
2827 let worker = std::thread::spawn(move || {
2828 started_tx.send(()).unwrap();
2829 assert!(
2830 result_tx
2831 .send(take_batch(&worker_shared, config, true))
2832 .is_ok()
2833 );
2834 });
2835 started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2836 let early = result_rx.recv_timeout(Duration::from_millis(100));
2837 let returned_while_idle = early.is_ok();
2838 {
2839 let mut q = shared.queue.lock().unwrap();
2840 if shutdown {
2841 q.shutdown = true;
2842 } else {
2843 let payload = Bytes::from_static(b"first-after-idle");
2844 q.bytes = payload.len();
2845 q.bytes_in_use = payload.len();
2846 q.next_seq = 2;
2847 q.last_enqueue = Some(Instant::now());
2848 q.oldest_enqueue = q.last_enqueue;
2849 q.force_flush = true;
2850 q.items.push_back(Pending {
2851 partition: PartitionKey("p".into()),
2852 record_count: 1,
2853 payload,
2854 meta: (),
2855 epoch: 0,
2856 durability: Durability::Buffered,
2857 responder: None,
2858 seq: 1,
2859 owner: None,
2860 expected_head: None,
2861 });
2862 }
2863 shared.cv.notify_all();
2864 }
2865 let result = early
2866 .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2867 .unwrap();
2868 worker.join().unwrap();
2869 assert!(
2870 !returned_while_idle,
2871 "idle worker polled with linger={linger:?}"
2872 );
2873 match result {
2874 TakeBatch::Shutdown => assert!(shutdown),
2875 TakeBatch::Batch(items) => {
2876 assert!(!shutdown);
2877 assert_eq!(items.len(), 1);
2878 assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2879 assert_eq!(items[0].seq, 1);
2880 }
2881 TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2882 }
2883 }
2884 }
2885 }
2886}
2887
2888#[cfg(test)]
2889mod append_owner_tests {
2890 use super::*;
2891 use crate::{InMemorySequencer, MemoryBlobStore};
2892 use async_trait::async_trait;
2893 use std::sync::atomic::{AtomicUsize, Ordering};
2894 struct Lifetime(Arc<AtomicUsize>);
2895 impl Drop for Lifetime {
2896 fn drop(&mut self) {
2897 self.0.fetch_add(1, Ordering::SeqCst);
2898 }
2899 }
2900 struct CopiedPausedPut {
2901 inner: MemoryBlobStore,
2902 entered: std::sync::mpsc::Sender<usize>,
2903 release: tokio::sync::watch::Sender<bool>,
2904 }
2905 #[async_trait]
2906 impl BlobStore for CopiedPausedPut {
2907 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
2908 self.put_chunks(key, vec![value]).await
2909 }
2910 async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
2911 let copied = Bytes::from(
2912 chunks
2913 .iter()
2914 .flat_map(|chunk| chunk.iter().copied())
2915 .collect::<Vec<_>>(),
2916 );
2917 drop(chunks);
2918 let mut release = self.release.subscribe();
2919 self.entered.send(copied.len()).unwrap();
2920 release.wait_for(|ready| *ready).await.unwrap();
2921 self.inner.put(key, copied).await
2922 }
2923 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
2924 self.inner.get(key).await
2925 }
2926 async fn get_range(
2927 &self,
2928 key: &str,
2929 range: std::ops::Range<u64>,
2930 ) -> Result<Option<Bytes>, ObjectLogError> {
2931 self.inner.get_range(key, range).await
2932 }
2933 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
2934 self.inner.list(prefix).await
2935 }
2936 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
2937 self.inner.delete(key).await
2938 }
2939 }
2940 struct ReentrantLifetime {
2941 shared: std::sync::Weak<Shared<()>>,
2942 unlocked: Arc<AtomicUsize>,
2943 }
2944 impl Drop for ReentrantLifetime {
2945 fn drop(&mut self) {
2946 let shared = self.shared.upgrade().unwrap();
2947 let available = !matches!(
2948 shared.queue.try_lock(),
2949 Err(std::sync::TryLockError::WouldBlock)
2950 );
2951 self.unlocked
2952 .store(usize::from(available), Ordering::SeqCst);
2953 }
2954 }
2955 #[test]
2956 fn rejected_queued_owner_drops_outside_engine_lock() {
2957 let config = FlushConfig::default();
2958 let shared = Arc::new(Shared {
2959 queue: Mutex::new(Queue {
2960 items: VecDeque::new(),
2961 bytes: 0,
2962 bytes_in_use: 0,
2963 shutdown: false,
2964 next_seq: 1,
2965 completed_through: 0,
2966 first_failed: None,
2967 force_flush: false,
2968 flush_waiters: Vec::new(),
2969 last_enqueue: None,
2970 oldest_enqueue: None,
2971 }),
2972 cv: Condvar::new(),
2973 max_buffered_bytes: config.max_buffered_bytes,
2974 budget: Mutex::new(BudgetRuntime::new(config.budget)),
2975 flush_config: config,
2976 });
2977 let unlocked = Arc::new(AtomicUsize::new(0));
2978 let owner = Arc::new(ReentrantLifetime {
2979 shared: Arc::downgrade(&shared),
2980 unlocked: unlocked.clone(),
2981 });
2982 shared.queue.lock().unwrap().items.push_back(Pending {
2983 partition: PartitionKey("p".into()),
2984 record_count: 1,
2985 payload: Bytes::from_static(b"queued"),
2986 meta: (),
2987 epoch: 0,
2988 durability: Durability::Sequenced,
2989 responder: None,
2990 seq: 1,
2991 owner: Some(owner),
2992 expected_head: None,
2993 });
2994 let uploads = fail_engine_and_take_pending_uploads(
2995 &shared,
2996 &mut VecDeque::new(),
2997 1,
2998 ObjectLogError::Sequencer("failure".into()),
2999 );
3000 assert!(uploads.is_empty());
3001 assert_eq!(unlocked.load(Ordering::SeqCst), 1);
3002 }
3003 #[test]
3004 fn failed_engine_keeps_owner_until_copied_pending_upload_finishes() {
3005 let runtime = tokio::runtime::Runtime::new().unwrap();
3006 let (entered, started) = std::sync::mpsc::channel();
3007 let blob = Arc::new(CopiedPausedPut {
3008 inner: MemoryBlobStore::new(),
3009 entered,
3010 release: tokio::sync::watch::channel(false).0,
3011 });
3012 let erased: Arc<dyn BlobStore> = blob.clone();
3013 let engine = LogEngine::new(
3014 erased.clone(),
3015 Arc::new(InMemorySequencer::new()),
3016 FlushConfig::default(),
3017 "idle/",
3018 );
3019 let drops = Arc::new(AtomicUsize::new(0));
3020 let (tx, mut rx) = oneshot::channel();
3021 let batch = vec![
3022 Pending {
3023 partition: PartitionKey("p".into()),
3024 record_count: 1,
3025 payload: Bytes::from_static(b"first"),
3026 meta: (),
3027 epoch: 0,
3028 durability: Durability::Sequenced,
3029 responder: Some(tx),
3030 seq: 1,
3031 owner: Some(Arc::new(Lifetime(drops.clone()))),
3032 expected_head: None,
3033 },
3034 Pending {
3035 partition: PartitionKey("p".into()),
3036 record_count: 1,
3037 payload: Bytes::from_static(b"second"),
3038 meta: (),
3039 epoch: 0,
3040 durability: Durability::Sequenced,
3041 responder: None,
3042 seq: 2,
3043 owner: None,
3044 expected_head: None,
3045 },
3046 ];
3047 let work = start_flush_work(&runtime, &erased, "data/", "", 1, batch, true);
3048 assert_eq!(started.recv_timeout(Duration::from_secs(5)).unwrap(), 11);
3049 let mut pending = VecDeque::from([work]);
3050 let uploads = fail_engine_and_take_pending_uploads(
3051 &engine.shared,
3052 &mut pending,
3053 1,
3054 ObjectLogError::Sequencer("injected committer failure".into()),
3055 );
3056 assert!(rx.try_recv().unwrap().is_err());
3057 assert!(pending.is_empty());
3058 assert_eq!(uploads.len(), 1);
3059 assert_eq!(drops.load(Ordering::SeqCst), 0);
3060 blob.release.send_replace(true);
3061 runtime.block_on(async {
3062 for upload in uploads {
3063 upload.await.unwrap().unwrap();
3064 }
3065 });
3066 assert_eq!(drops.load(Ordering::SeqCst), 1);
3067 assert_eq!(
3068 runtime
3069 .block_on(blob.inner.get("data/00000000000000000001"))
3070 .unwrap()
3071 .unwrap(),
3072 Bytes::from_static(b"firstsecond")
3073 );
3074 }
3075 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
3076 async fn cancelled_producer_keeps_owner_until_copied_upload_and_flush_finish() {
3077 let (entered, started) = std::sync::mpsc::channel();
3078 let blob = Arc::new(CopiedPausedPut {
3079 inner: MemoryBlobStore::new(),
3080 entered,
3081 release: tokio::sync::watch::channel(false).0,
3082 });
3083 let engine = Arc::new(LogEngine::new(
3084 blob.clone(),
3085 Arc::new(InMemorySequencer::new()),
3086 FlushConfig {
3087 linger: Duration::ZERO,
3088 ..Default::default()
3089 },
3090 "data/",
3091 ));
3092 let drops = Arc::new(AtomicUsize::new(0));
3093 let owner: AppendOwner = Arc::new(Lifetime(drops.clone()));
3094 let writer = engine.clone();
3095 let caller = tokio::spawn(async move {
3096 writer
3097 .produce_owned_at_epoch(
3098 PartitionKey("p".into()),
3099 Bytes::from_static(b"payload"),
3100 1,
3101 (),
3102 Durability::Sequenced,
3103 0,
3104 owner,
3105 )
3106 .await
3107 });
3108 assert_eq!(started.recv_timeout(Duration::from_secs(5)).unwrap(), 7);
3109 caller.abort();
3110 let _ = caller.await;
3111 assert_eq!(drops.load(Ordering::SeqCst), 0);
3112 blob.release.send_replace(true);
3113 engine.flush().await.unwrap();
3114 tokio::time::timeout(Duration::from_secs(5), async {
3115 while drops.load(Ordering::SeqCst) == 0 {
3116 tokio::task::yield_now().await;
3117 }
3118 })
3119 .await
3120 .unwrap();
3121 assert_eq!(drops.load(Ordering::SeqCst), 1);
3122 }
3123}