1use crate::budget::{
4 BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24 Buffered,
27 Durable,
30 Sequenced,
33}
34
35#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38 pub max_bytes: usize,
43 pub max_batches: usize,
45 pub linger: Duration,
51 pub max_inflight_flushes: usize,
55 pub max_buffered_bytes: usize,
58 pub budget: BudgetConfig,
60}
61
62impl Default for FlushConfig {
63 fn default() -> Self {
64 Self {
65 max_bytes: 1024 * 1024 * 1024,
67 max_batches: 10_000,
68 linger: Duration::from_millis(50),
69 max_inflight_flushes: 1,
71 max_buffered_bytes: 2 * 1024 * 1024 * 1024,
72 budget: BudgetConfig::default(),
73 }
74 }
75}
76
77#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct AppendOutcome {
80 pub base_offset: Option<i64>,
82 pub last_offset: Option<i64>,
84 pub durable: bool,
86 pub sequenced: bool,
88}
89
90#[derive(Clone, Debug, PartialEq, Eq)]
92pub struct FetchedBatch {
93 pub base_offset: i64,
95 pub record_count: i32,
97 pub payload: Bytes,
99}
100
101#[derive(Clone, Debug, PartialEq, Eq)]
103pub struct BufferStats {
104 pub queued_bytes: usize,
106 pub bytes_in_use: usize,
108 pub queued_batches: usize,
110 pub max_buffered_bytes: usize,
112}
113
114type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
115
116fn retryable_storage_error(err: &ObjectLogError) -> bool {
117 matches!(err, ObjectLogError::StorageUnavailable(_))
118}
119
120async fn retry_delay(attempt: usize) {
121 let multiplier = 1u32 << attempt.min(5);
122 tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
123}
124
125async fn put_chunks_with_retries(
126 blob: &Arc<dyn BlobStore>,
127 key: &str,
128 chunks: Vec<Bytes>,
129) -> Result<(), ObjectLogError> {
130 let mut attempt = 0usize;
132 loop {
133 match blob.put_chunks(key, chunks.clone()).await {
134 Ok(()) => return Ok(()),
135 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
136 retry_delay(attempt).await;
137 attempt += 1;
138 }
139 Err(err) => return Err(err),
140 }
141 }
142}
143
144async fn get_range_with_retries(
145 blob: &Arc<dyn BlobStore>,
146 key: &str,
147 range: std::ops::Range<u64>,
148) -> Result<Option<Bytes>, ObjectLogError> {
149 let mut attempt = 0usize;
150 loop {
151 match blob.get_range(key, range.clone()).await {
152 Ok(bytes) => return Ok(bytes),
153 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
154 retry_delay(attempt).await;
155 attempt += 1;
156 }
157 Err(err) => return Err(err),
158 }
159 }
160}
161
162struct Pending<M> {
163 partition: PartitionKey,
164 record_count: i32,
165 payload: Bytes,
166 meta: M,
167 epoch: u64,
169 durability: Durability,
170 responder: Option<Responder>,
171 seq: u64,
173}
174
175struct FlushWork<M> {
176 batch: Vec<Pending<M>>,
177 locations: Vec<BatchLocation>,
178 responders: Vec<(Durability, Option<Responder>)>,
179 bytes: usize,
180 max_seq: u64,
182 data_objects: usize,
184 put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
185 put_started: Instant,
186 put_result: Option<Result<Duration, ObjectLogError>>,
187}
188
189struct CommitJob {
190 task: TokioJoinHandle<usize>,
191 first_seq: u64,
192}
193
194enum TakeBatch<M> {
195 Batch(Vec<Pending<M>>),
196 Empty,
197 Shutdown,
198}
199
200type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
201
202struct Queue<M> {
203 items: VecDeque<Pending<M>>,
204 bytes: usize,
205 bytes_in_use: usize,
206 shutdown: bool,
207 next_seq: u64,
209 completed_through: u64,
211 first_failed: Option<(u64, ObjectLogError)>,
213 force_flush: bool,
215 flush_waiters: Vec<FlushWaiter>,
217 last_enqueue: Option<Instant>,
219 oldest_enqueue: Option<Instant>,
221}
222
223struct Shared<M> {
224 queue: Mutex<Queue<M>>,
225 cv: Condvar,
226 max_buffered_bytes: usize,
227 budget: Mutex<BudgetRuntime>,
229 flush_config: FlushConfig,
231}
232
233pub struct LogEngine<S: Sequencer> {
242 shared: Arc<Shared<S::Meta>>,
243 blob: Arc<dyn BlobStore>,
244 sequencer: Arc<S>,
245 key_prefix: String,
248 writer_id: String,
249 flush_thread: Option<JoinHandle<()>>,
250}
251
252impl<S> LogEngine<S>
253where
254 S: Sequencer + 'static,
255 S::Meta: Send + 'static,
256{
257 pub fn new(
261 blob: Arc<dyn BlobStore>,
262 sequencer: Arc<S>,
263 config: FlushConfig,
264 key_prefix: impl Into<String>,
265 ) -> Self {
266 Self::new_with_writer(blob, sequencer, config, key_prefix, "")
267 }
268
269 pub fn new_with_writer(
273 blob: Arc<dyn BlobStore>,
274 sequencer: Arc<S>,
275 config: FlushConfig,
276 key_prefix: impl Into<String>,
277 writer_id: impl Into<String>,
278 ) -> Self {
279 if let Err(msg) = config.budget.validate() {
280 panic!("invalid FlushConfig.budget: {msg}");
281 }
282 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
283 eprintln!("object-log flush config: {config:?}");
284 }
285 let budget_rt = BudgetRuntime::new(config.budget);
286 let shared = Arc::new(Shared {
287 queue: Mutex::new(Queue {
288 items: VecDeque::new(),
289 bytes: 0,
290 bytes_in_use: 0,
291 shutdown: false,
292 next_seq: 1, completed_through: 0,
294 first_failed: None,
295 force_flush: false,
296 flush_waiters: Vec::new(),
297 last_enqueue: None,
298 oldest_enqueue: None,
299 }),
300 cv: Condvar::new(),
301 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
302 budget: Mutex::new(budget_rt),
303 flush_config: config,
304 });
305 let key_prefix = key_prefix.into();
306 let writer_id = writer_id.into();
307 let flush_thread = {
308 let shared = Arc::clone(&shared);
309 let blob = Arc::clone(&blob);
310 let sequencer = Arc::clone(&sequencer);
311 let prefix = key_prefix.clone();
312 let writer_id = writer_id.clone();
313 std::thread::Builder::new()
314 .name("object-log-flush".into())
315 .spawn(move || flush_loop(shared, blob, sequencer, config, prefix, writer_id))
316 .expect("spawn flush thread")
317 };
318 Self {
319 writer_id,
320 shared,
321 blob,
322 sequencer,
323 key_prefix,
324 flush_thread: Some(flush_thread),
325 }
326 }
327
328 pub fn data_prefix(&self) -> &str {
330 &self.key_prefix
331 }
332
333 pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
335 &self.blob
336 }
337
338 pub fn sequencer(&self) -> &Arc<S> {
340 &self.sequencer
341 }
342
343 pub fn writer_id(&self) -> &str {
345 &self.writer_id
346 }
347
348 pub async fn produce(
353 &self,
354 partition: PartitionKey,
355 payload: Bytes,
356 record_count: i32,
357 meta: S::Meta,
358 durability: Durability,
359 ) -> Result<AppendOutcome, ObjectLogError> {
360 self.produce_at_epoch(partition, payload, record_count, meta, durability, 0)
361 .await
362 }
363
364 pub async fn produce_at_epoch(
366 &self,
367 partition: PartitionKey,
368 payload: Bytes,
369 record_count: i32,
370 meta: S::Meta,
371 durability: Durability,
372 epoch: u64,
373 ) -> Result<AppendOutcome, ObjectLogError> {
374 if record_count <= 0 {
375 return Err(ObjectLogError::InvalidBatch(
376 "record_count must be > 0".into(),
377 ));
378 }
379 if matches!(durability, Durability::Buffered) {
380 self.enqueue(Pending {
381 partition,
382 record_count,
383 payload,
384 meta,
385 epoch,
386 durability,
387 responder: None,
388 seq: 0, })?;
390 return Ok(AppendOutcome {
391 base_offset: None,
392 last_offset: None,
393 durable: false,
394 sequenced: false,
395 });
396 }
397 {
399 let mut budget = self.shared.budget.lock().expect("poisoned");
400 if budget.config.enabled
401 && budget.config.mode == BudgetMode::FailClosed
402 && !budget.reserve_for_fail_closed(Instant::now())
403 {
404 return Err(ObjectLogError::BudgetExceeded(
405 "insufficient durable-ops budget to admit produce".into(),
406 ));
407 }
408 }
409 if self.shared.flush_config.budget.enabled
411 && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
412 {
413 let timeout = self.shared.flush_config.budget.admission_timeout;
414 let deadline = Instant::now() + timeout;
415 loop {
416 let mut budget = self.shared.budget.lock().expect("poisoned");
417 if budget.can_admit_now(Instant::now()) {
418 break;
419 }
420 drop(budget);
421 if Instant::now() >= deadline {
422 return Err(ObjectLogError::BudgetExceeded(
423 "timed out waiting for durable-ops budget".into(),
424 ));
425 }
426 std::thread::sleep(Duration::from_millis(1));
427 }
428 }
429 let (tx, rx) = oneshot::channel();
430 self.enqueue(Pending {
431 partition,
432 record_count,
433 payload,
434 meta,
435 epoch,
436 durability,
437 responder: Some(tx),
438 seq: 0,
439 })?;
440 rx.await
441 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
442 }
443
444 pub async fn flush(&self) -> Result<(), ObjectLogError> {
453 let rx = {
454 let mut q = self.shared.queue.lock().expect("poisoned");
455 if q.shutdown {
456 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
457 }
458 if q.next_seq == 1 {
459 return Ok(());
461 }
462 let barrier = q.next_seq - 1;
463 if let Some((sequence, error)) = &q.first_failed {
464 if *sequence <= barrier {
465 return Err(error.clone());
466 }
467 }
468 if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
469 return Ok(());
470 }
471 let (tx, rx) = oneshot::channel();
472 q.force_flush = true;
473 q.flush_waiters.push((barrier, tx));
474 self.shared.cv.notify_all();
475 rx
476 };
477 rx.await
478 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
479 }
480
481 fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
482 let item_len = item.payload.len();
483 let max_buffered_bytes = self.shared.max_buffered_bytes;
484 let shared = Arc::clone(&self.shared);
485 let mut q = shared.queue.lock().expect("poisoned");
486 while !q.shutdown
487 && q.bytes_in_use > 0
488 && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
489 {
490 q = shared.cv.wait(q).expect("poisoned");
491 }
492 if q.shutdown {
493 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
494 }
495 let now = Instant::now();
496 item.seq = q.next_seq;
497 q.next_seq = q.next_seq.saturating_add(1);
498 q.bytes += item_len;
499 q.bytes_in_use += item_len;
500 q.last_enqueue = Some(now);
501 if q.oldest_enqueue.is_none() {
502 q.oldest_enqueue = Some(now);
503 }
504 q.items.push_back(item);
505 shared.cv.notify_all();
506 Ok(())
507 }
508
509 pub async fn fetch(
511 &self,
512 partition: &PartitionKey,
513 offset: i64,
514 max_bytes: usize,
515 ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
516 let entries = self.sequencer.lookup(partition, offset)?;
517 let mut out = Vec::new();
518 let mut total = 0usize;
519 for e in entries {
520 if total >= max_bytes && !out.is_empty() {
521 break;
522 }
523 let batch = self.load_entry(&e).await?;
524 total += batch.payload.len();
525 out.push(batch);
526 }
527 Ok(out)
528 }
529
530 pub async fn fetch_stream<F>(
537 &self,
538 partition: &PartitionKey,
539 offset: i64,
540 mut visit: F,
541 ) -> Result<(), ObjectLogError>
542 where
543 F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
544 {
545 let entries = self.sequencer.lookup(partition, offset)?;
546 for e in entries {
547 visit(self.load_entry(&e).await?)?;
548 }
549 Ok(())
550 }
551
552 async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
553 let start = e.location.byte_start as u64;
554 let end = start + e.location.byte_len as u64;
555 let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
556 .await?
557 .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
558 Ok(FetchedBatch {
559 base_offset: e.base_offset,
560 record_count: e.record_count,
561 payload: bytes,
562 })
563 }
564
565 pub async fn truncate_before(
568 &self,
569 partition: &PartitionKey,
570 offset: i64,
571 ) -> Result<(), ObjectLogError> {
572 let dead = self.sequencer.truncate_before(partition, offset)?;
573 for object_id in dead {
574 self.blob.delete(&object_id).await?;
575 }
576 Ok(())
577 }
578
579 pub async fn reap_orphans(
589 &self,
590 live: &HashSet<String>,
591 ) -> Result<Vec<String>, ObjectLogError> {
592 reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
593 }
594
595 pub fn buffer_stats(&self) -> BufferStats {
597 let q = self.shared.queue.lock().expect("poisoned");
598 BufferStats {
599 queued_bytes: q.bytes,
600 bytes_in_use: q.bytes_in_use,
601 queued_batches: q.items.len(),
602 max_buffered_bytes: self.shared.max_buffered_bytes,
603 }
604 }
605
606 pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
608 let now = Instant::now();
609 let mut budget = self.shared.budget.lock().expect("poisoned");
610 budget.refill(now);
611 let effective = budget.effective_budget_per_sec;
612 let reason = if !budget.config.enabled {
613 EffectiveReason::Disabled
614 } else if budget.config.budget_per_sec_cap.is_some()
615 && budget
616 .config
617 .budget_per_sec_cap
618 .is_some_and(|c| (effective - c).abs() < 1e-9)
619 {
620 EffectiveReason::ConfigCap
621 } else if budget.ongoing_capacity.is_some() {
622 EffectiveReason::Ongoing
623 } else if budget.startup_capacity.is_some() {
624 EffectiveReason::StartupProbe
625 } else {
626 EffectiveReason::DefaultCapacity
627 };
628 let (queued, last_enq) = {
629 let q = self.shared.queue.lock().expect("poisoned");
630 (q.bytes, q.last_enqueue)
631 };
632 let early = budget.allow_early_flush(now, queued, last_enq);
633 let max_linger = self.shared.flush_config.linger;
634 let eff_linger = if !budget.config.enabled {
635 max_linger
636 } else if early || max_linger.is_zero() {
637 Duration::ZERO
638 } else {
639 max_linger
640 };
641 PipelineSnapshot {
642 budget_per_sec: EffectiveKnob {
643 configured: budget.config.budget_per_sec_cap,
644 startup_measured: budget.startup_capacity,
645 ongoing_measured: budget.ongoing_capacity,
646 effective,
647 reason,
648 },
649 effective_linger_ms: EffectiveKnob {
650 configured: Some(max_linger.as_millis() as u64),
651 startup_measured: None,
652 ongoing_measured: None,
653 effective: eff_linger.as_millis() as u64,
654 reason: if early {
655 EffectiveReason::Ongoing
656 } else {
657 EffectiveReason::Configured
658 },
659 },
660 max_linger_ms: max_linger.as_millis() as u64,
661 token_fill_ratio: budget.fill_ratio(),
662 media_ops_total: budget.media_ops_total,
663 overdraft_total: budget.overdraft_total,
664 flushes_total: budget.flushes_total,
665 undersized_deadline_flushes: budget.undersized_deadline_flushes,
666 predicted_media_ops: budget.predicted_media_ops,
667 budget_enabled: budget.config.enabled,
668 budget_mode: budget.config.mode,
669 }
670 }
671}
672
673pub async fn reap_orphans(
679 blob: &dyn BlobStore,
680 data_prefix: &str,
681 live: &HashSet<String>,
682) -> Result<Vec<String>, ObjectLogError> {
683 let keys = blob.list(data_prefix).await?;
684 let mut deleted = Vec::new();
685 for key in keys {
686 if live.contains(&key) {
687 continue;
688 }
689 blob.delete(&key).await?;
690 deleted.push(key);
691 }
692 Ok(deleted)
693}
694
695impl<S: Sequencer> Drop for LogEngine<S> {
696 fn drop(&mut self) {
697 {
698 let mut q = self.shared.queue.lock().expect("poisoned");
699 q.shutdown = true;
700 }
701 self.shared.cv.notify_all();
702 if let Some(t) = self.flush_thread.take() {
703 let _ = t.join();
704 }
705 }
706}
707
708async fn recover_data_object_counter(
714 blob: &dyn BlobStore,
715 prefix: &str,
716 writer_id: &str,
717) -> Result<u64, ObjectLogError> {
718 let keys = blob.list(prefix).await?;
719 let mut max = 0u64;
720 let writer_prefix = format!("{prefix}{writer_id}/");
721 for key in keys {
722 let Some(suffix) = key.strip_prefix(prefix) else {
723 continue;
724 };
725 if writer_id.is_empty() {
726 if let Ok(n) = suffix.parse::<u64>() {
727 max = max.max(n);
728 }
729 continue;
730 }
731 let Some(rest) = key.strip_prefix(&writer_prefix) else {
732 continue;
733 };
734 if let Some(n) = rest
735 .rsplit('/')
736 .next()
737 .and_then(|tail| tail.parse::<u64>().ok())
738 {
739 max = max.max(n);
740 }
741 }
742 Ok(max)
743}
744
745fn data_object_key(prefix: &str, writer_id: &str, epoch: u64, counter: u64) -> String {
746 if writer_id.is_empty() {
747 format!("{prefix}{counter:020}")
748 } else {
749 format!("{prefix}{writer_id}/{epoch:020}/{counter:020}")
750 }
751}
752
753fn flush_loop<S>(
754 shared: Arc<Shared<S::Meta>>,
755 blob: Arc<dyn BlobStore>,
756 sequencer: Arc<S>,
757 config: FlushConfig,
758 prefix: String,
759 writer_id: String,
760) where
761 S: Sequencer + 'static,
762 S::Meta: Send + 'static,
763{
764 let max_inflight = config.max_inflight_flushes.max(1);
765 let commit_group_limit = if sequencer.supports_multi_object_commit() {
766 max_inflight
767 } else {
768 1
769 };
770 let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
771 .ok()
772 .and_then(|v| v.parse::<usize>().ok())
773 .filter(|v| *v > 0)
774 .unwrap_or_else(|| max_inflight.min(8));
775 let rt = tokio::runtime::Builder::new_multi_thread()
776 .worker_threads(worker_threads)
777 .enable_all()
778 .build()
779 .expect("flush runtime");
780 let mut counter = match rt.block_on(recover_data_object_counter(
784 blob.as_ref(),
785 &prefix,
786 &writer_id,
787 )) {
788 Ok(counter) => counter,
789 Err(error) => {
790 let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
793 return;
794 }
795 };
796 let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
797 let mut active_puts = 0usize;
798 let mut committing: Option<CommitJob> = None;
799 let mut shutdown = false;
800
801 loop {
802 while !shutdown && active_puts < max_inflight {
803 let wait_for_more = pending.is_empty() && committing.is_none();
806 match take_batch(&shared, config, wait_for_more) {
807 TakeBatch::Batch(batch) => {
808 counter += 1;
809 let concurrent = max_inflight > 1;
810 let work = start_flush_work(
811 &rt, &blob, &prefix, &writer_id, counter, batch, concurrent,
812 );
813 if concurrent {
814 active_puts += 1;
815 }
816 pending.push_back(work);
817 }
818 TakeBatch::Empty => break,
819 TakeBatch::Shutdown => shutdown = true,
820 }
821 }
822
823 let mut made_progress = false;
824 for work in pending.iter_mut() {
825 let Some(put) = work.put.as_ref() else {
826 continue;
827 };
828 if !put.is_finished() {
829 continue;
830 }
831 let put = work.put.take().expect("put handle exists");
832 let elapsed = work.put_started.elapsed();
833 work.put_result = Some(match rt.block_on(put) {
834 Ok(Ok(())) => Ok(elapsed),
835 Ok(Err(e)) => Err(e),
836 Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
837 "flush task failed: {e}"
838 ))),
839 });
840 active_puts = active_puts.saturating_sub(1);
841 made_progress = true;
842 }
843
844 if committing
845 .as_ref()
846 .is_some_and(|job| job.task.is_finished())
847 {
848 let job = committing.take().expect("completed commit exists");
849 match rt.block_on(job.task) {
850 Ok(released) => {
851 let mut q = shared.queue.lock().expect("poisoned");
852 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
853 shared.cv.notify_all();
854 }
855 Err(error) => {
856 let uploads = fail_engine_and_take_pending_uploads(
857 &shared,
858 &mut pending,
859 job.first_seq,
860 ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
861 );
862 for upload in uploads {
866 let _ = rt.block_on(upload);
867 }
868 return;
869 }
870 }
871 made_progress = true;
872 }
873
874 if committing.is_none() {
875 if let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit) {
876 if max_inflight == 1 {
877 let released = finish_flush_work(&shared, &blob, &sequencer, work);
879 let mut q = shared.queue.lock().expect("poisoned");
880 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
881 shared.cv.notify_all();
882 } else {
883 let first_seq = work
886 .batch
887 .iter()
888 .map(|p| p.seq)
889 .min()
890 .unwrap_or(work.max_seq);
891 let shared = shared.clone();
892 let blob = blob.clone();
893 let sequencer = sequencer.clone();
894 committing = Some(CommitJob {
895 task: rt.spawn_blocking(move || {
896 finish_flush_work(&shared, &blob, &sequencer, work)
897 }),
898 first_seq,
899 });
900 }
901 made_progress = true;
902 }
903 }
904
905 if pending.is_empty() && committing.is_none() {
906 {
907 let mut q = shared.queue.lock().expect("poisoned");
908 if q.items.is_empty() && q.bytes_in_use == 0 {
909 notify_flush_waiters(&mut q);
912 }
913 }
914 shared.cv.notify_all();
915 if shutdown {
916 let mut q = shared.queue.lock().expect("poisoned");
918 for (_, tx) in q.flush_waiters.drain(..) {
919 let _ = tx.send(Err(ObjectLogError::Sequencer(
920 "engine shutting down".into(),
921 )));
922 }
923 return;
924 }
925 continue;
926 }
927
928 if !made_progress {
929 std::thread::sleep(Duration::from_millis(1));
930 }
931 }
932}
933
934fn fail_engine_and_take_pending_uploads<M>(
937 shared: &Arc<Shared<M>>,
938 pending: &mut VecDeque<FlushWork<M>>,
939 first_seq: u64,
940 error: ObjectLogError,
941) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
942 let mut uploads = Vec::new();
943 for mut work in pending.drain(..) {
944 if let Some(put) = work.put.take() {
945 uploads.push(put);
946 }
947 send_storage_error(&mut work.responders, error.clone());
948 }
949 let mut q = shared
951 .queue
952 .lock()
953 .unwrap_or_else(|poisoned| poisoned.into_inner());
954 q.shutdown = true;
955 q.first_failed.get_or_insert((first_seq, error.clone()));
956 for mut item in q.items.drain(..) {
957 if let Some(tx) = item.responder.take() {
958 let _ = tx.send(Err(error.clone()));
959 }
960 }
961 q.bytes = 0;
962 for (_, tx) in q.flush_waiters.drain(..) {
965 let _ = tx.send(Err(error.clone()));
966 }
967 q.force_flush = false;
968 shared.cv.notify_all();
969 uploads
970}
971
972fn take_ready_commit_group<M>(
975 pending: &mut VecDeque<FlushWork<M>>,
976 max_objects: usize,
977) -> Option<FlushWork<M>> {
978 pending.front()?.put_result.as_ref()?;
979 let mut group = pending.pop_front().expect("ready front exists");
980 if group.put_result.as_ref().is_some_and(Result::is_err) {
981 return Some(group);
982 }
983 while group.data_objects < max_objects.max(1) {
984 let Some(next) = pending.front() else { break };
985 if !next.put_result.as_ref().is_some_and(Result::is_ok)
986 || group.data_objects.saturating_add(next.data_objects) > max_objects
987 {
988 break;
989 }
990 let mut next = pending.pop_front().expect("ready successor exists");
991 group.batch.append(&mut next.batch);
992 group.locations.append(&mut next.locations);
993 group.responders.append(&mut next.responders);
994 group.bytes = group
995 .bytes
996 .checked_add(next.bytes)
997 .expect("bounded pending bytes");
998 group.max_seq = group.max_seq.max(next.max_seq);
999 group.data_objects += next.data_objects;
1000 let next_elapsed = next.put_result.take().unwrap().unwrap();
1001 let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
1002 *elapsed = (*elapsed).max(next_elapsed);
1003 }
1004 Some(group)
1005}
1006
1007fn effective_linger<M>(
1011 shared: &Shared<M>,
1012 config: FlushConfig,
1013 queued_bytes: usize,
1014 last_enqueue: Option<Instant>,
1015 oldest_enqueue: Option<Instant>,
1016) -> Duration {
1017 if config.linger.is_zero() {
1018 return Duration::ZERO;
1019 }
1020 let now = Instant::now();
1021 if config.budget.enabled {
1022 let mut budget = shared.budget.lock().expect("poisoned");
1023 budget.refill(now);
1024 if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
1025 budget.note_early_flush(now);
1026 return Duration::ZERO;
1027 }
1028 }
1029 if let Some(oldest) = oldest_enqueue {
1031 let deadline = oldest + config.linger;
1032 if now >= deadline {
1033 return Duration::ZERO;
1034 }
1035 let until_deadline = deadline.saturating_duration_since(now);
1036 if config.budget.enabled
1038 && !config.budget.early_flush_idle.is_zero()
1039 && let Some(last) = last_enqueue
1040 {
1041 let idle_at = last + config.budget.early_flush_idle;
1042 if idle_at > now && idle_at < deadline {
1043 return idle_at.saturating_duration_since(now);
1044 }
1045 }
1046 return until_deadline;
1047 }
1048 config.linger
1049}
1050
1051fn take_batch<M>(
1052 shared: &Arc<Shared<M>>,
1053 config: FlushConfig,
1054 wait_when_empty: bool,
1055) -> TakeBatch<M> {
1056 let mut q = shared.queue.lock().expect("poisoned");
1057 loop {
1058 if q.items.is_empty() {
1059 if q.shutdown {
1060 return TakeBatch::Shutdown;
1061 }
1062 if !wait_when_empty {
1063 return TakeBatch::Empty;
1064 }
1065 q = shared.cv.wait(q).expect("poisoned");
1069 continue;
1070 }
1071
1072 let linger = {
1073 let queued = q.bytes;
1074 let last_enq = q.last_enqueue;
1075 let oldest = q.oldest_enqueue;
1076 drop(q);
1077 let l = effective_linger(shared, config, queued, last_enq, oldest);
1078 q = shared.queue.lock().expect("poisoned");
1079 l
1080 };
1081
1082 let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1083 let force = q.force_flush;
1084 let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1085 if triggered {
1086 if !size_trigger && !q.shutdown && !q.items.is_empty() {
1087 let mut budget = shared.budget.lock().expect("poisoned");
1088 if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1089 budget.undersized_deadline_flushes += 1;
1090 }
1091 }
1092 break;
1093 }
1094
1095 let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1098 q = guard;
1099 }
1100
1101 let force_drain = q.force_flush;
1102 let mut items = Vec::new();
1103 let mut bytes = 0usize;
1104 while let Some(item) = q.items.pop_front() {
1107 q.bytes = q.bytes.saturating_sub(item.payload.len());
1108 bytes += item.payload.len();
1109 items.push(item);
1110 if bytes >= config.max_bytes {
1111 break;
1112 }
1113 if !force_drain && items.len() >= config.max_batches {
1114 break;
1115 }
1116 }
1117 q.oldest_enqueue = if q.items.is_empty() {
1118 None
1119 } else {
1120 Some(Instant::now())
1122 };
1123 if force_drain {
1125 q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1126 }
1127 if items.is_empty() {
1128 TakeBatch::Empty
1129 } else {
1130 TakeBatch::Batch(items)
1131 }
1132}
1133
1134fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1135 let done = q.completed_through;
1136 let mut i = 0;
1137 while i < q.flush_waiters.len() {
1138 let barrier = q.flush_waiters[i].0;
1139 let failure = q
1140 .first_failed
1141 .as_ref()
1142 .filter(|(sequence, _)| *sequence <= barrier)
1143 .map(|(_, error)| error.clone());
1144 if barrier <= done || failure.is_some() {
1145 let (_, tx) = q.flush_waiters.swap_remove(i);
1146 let _ = tx.send(failure.map_or(Ok(()), Err));
1147 } else {
1148 i += 1;
1149 }
1150 }
1151 if q.flush_waiters.is_empty() {
1152 q.force_flush = false;
1153 } else if q.items.is_empty() && q.bytes_in_use == 0 {
1154 q.force_flush = true;
1156 } else {
1157 q.force_flush = true;
1158 }
1159}
1160
1161fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1162 for (_, tx) in responders.iter_mut() {
1163 if let Some(tx) = tx.take() {
1164 let _ = tx.send(Err(err.clone()));
1165 }
1166 }
1167}
1168
1169fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1170 for (durability, tx) in responders.iter_mut() {
1171 if *durability == Durability::Durable
1172 && let Some(tx) = tx.take()
1173 {
1174 let _ = tx.send(Ok(AppendOutcome {
1175 base_offset: None,
1176 last_offset: None,
1177 durable: true,
1178 sequenced: false,
1179 }));
1180 }
1181 }
1182}
1183
1184fn prepare_flush_work<M>(
1185 prefix: &str,
1186 writer_id: &str,
1187 counter: u64,
1188 mut batch: Vec<Pending<M>>,
1189) -> (FlushWork<M>, String, Vec<Bytes>) {
1190 let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1191 let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1192 let epoch = batch.first().map(|pending| pending.epoch).unwrap_or(0);
1193 let key = data_object_key(prefix, writer_id, epoch, counter);
1194 let mut offset = 0usize;
1195 for p in &batch {
1196 let start = offset as u32;
1197 offset += p.payload.len();
1198 chunks.push(p.payload.clone());
1199 locations.push(BatchLocation {
1200 object_id: key.clone(),
1201 byte_start: start,
1202 byte_len: p.payload.len() as u32,
1203 });
1204 }
1205 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1206 eprintln!(
1207 "object-log seal: key={key} batches={} bytes={offset}",
1208 batch.len()
1209 );
1210 }
1211 let responders: Vec<(Durability, Option<Responder>)> = batch
1212 .iter_mut()
1213 .map(|p| (p.durability, p.responder.take()))
1214 .collect();
1215 let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1216 let work = FlushWork {
1217 batch,
1218 locations,
1219 responders,
1220 bytes: offset,
1221 max_seq,
1222 data_objects: 1,
1223 put: None,
1224 put_started: Instant::now(),
1225 put_result: None,
1226 };
1227 (work, key, chunks)
1228}
1229
1230fn start_flush_work<M>(
1236 rt: &tokio::runtime::Runtime,
1237 blob: &Arc<dyn BlobStore>,
1238 prefix: &str,
1239 writer_id: &str,
1240 counter: u64,
1241 batch: Vec<Pending<M>>,
1242 concurrent: bool,
1243) -> FlushWork<M> {
1244 let (mut work, key, chunks) = prepare_flush_work(prefix, writer_id, counter, batch);
1245 let blob = Arc::clone(blob);
1246 let _ = blob.take_media_op_stats();
1247 work.put_started = Instant::now();
1248 if concurrent {
1249 work.put =
1250 Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
1251 } else {
1252 let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1253 let elapsed = work.put_started.elapsed();
1254 work.put_result = Some(match put_result {
1255 Ok(()) => Ok(elapsed),
1256 Err(e) => Err(e),
1257 });
1258 }
1259 work
1260}
1261
1262fn finish_flush_work<S, M>(
1263 shared: &Arc<Shared<M>>,
1264 blob: &Arc<dyn BlobStore>,
1265 sequencer: &Arc<S>,
1266 mut work: FlushWork<S::Meta>,
1267) -> usize
1268where
1269 S: Sequencer + 'static,
1270 S::Meta: Send + 'static,
1271{
1272 let release_bytes = work.bytes;
1273 let first_seq = work
1274 .batch
1275 .iter()
1276 .map(|pending| pending.seq)
1277 .min()
1278 .unwrap_or(work.max_seq);
1279 let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1283 let put_elapsed = match work.put_result.take().expect("put result is ready") {
1284 Ok(elapsed) => elapsed,
1285 Err(e) => {
1286 send_storage_error(&mut work.responders, e.clone());
1287 let mut q = shared.queue.lock().expect("poisoned");
1288 let done = work.max_seq;
1290 if done >= q.completed_through {
1291 q.completed_through = done;
1292 }
1293 q.first_failed.get_or_insert((first_seq, e));
1294 notify_flush_waiters(&mut q);
1295 shared.cv.notify_all();
1296 return release_bytes;
1297 }
1298 };
1299
1300 let mut media_ops = blob
1302 .take_media_op_stats()
1303 .map(|s| s.media_ops)
1304 .unwrap_or(work.data_objects as u64); send_durable_acks(&mut work.responders);
1308
1309 let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1311 .batch
1312 .iter()
1313 .zip(work.locations.iter())
1314 .map(|(p, loc)| CommitBatch {
1315 partition: p.partition.clone(),
1316 record_count: p.record_count,
1317 location: loc.clone(),
1318 meta: &p.meta,
1319 epoch: p.epoch,
1320 })
1321 .collect();
1322
1323 let commit_started = timing.then(Instant::now);
1324 let _ = blob.take_media_op_stats();
1327 let mut commit_error = None;
1328 match sequencer.commit(&commit_batches) {
1329 Ok(outcomes) => {
1330 if let Some(commit_started) = commit_started {
1331 eprintln!(
1332 "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1333 work.bytes,
1334 commit_batches.len(),
1335 put_elapsed.as_millis(),
1336 commit_started.elapsed().as_millis(),
1337 work.data_objects
1338 );
1339 }
1340 if let Some(s) = blob.take_media_op_stats() {
1341 media_ops = media_ops.saturating_add(s.media_ops);
1342 }
1343 for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1344 if let Some(tx) = tx.take() {
1345 let (base, last) = match outcome {
1346 CommitOutcome::Assigned {
1347 base_offset,
1348 record_count,
1349 } => (
1350 Some(base_offset),
1351 Some(base_offset + record_count as i64 - 1),
1352 ),
1353 CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1354 CommitOutcome::Rejected { reason } => {
1355 let _ = tx.send(Err(ObjectLogError::Sequencer(reason)));
1356 continue;
1357 }
1358 };
1359 let _ = tx.send(Ok(AppendOutcome {
1360 base_offset: base,
1361 last_offset: last,
1362 durable: true,
1363 sequenced: true,
1364 }));
1365 }
1366 }
1367 }
1368 Err(e) => {
1369 if let Some(s) = blob.take_media_op_stats() {
1370 media_ops = media_ops.saturating_add(s.media_ops);
1371 }
1372 send_storage_error(&mut work.responders, e.clone());
1373 commit_error = Some(e);
1374 }
1375 }
1376
1377 {
1378 let mut budget = shared.budget.lock().expect("poisoned");
1379 budget.consume_after_flush(media_ops, Instant::now());
1380 }
1381 {
1382 let mut q = shared.queue.lock().expect("poisoned");
1383 if let Some(error) = commit_error {
1384 q.first_failed.get_or_insert((first_seq, error));
1385 }
1386 if work.max_seq >= q.completed_through {
1387 q.completed_through = work.max_seq;
1388 }
1389 notify_flush_waiters(&mut q);
1390 }
1391 shared.cv.notify_all();
1392
1393 release_bytes
1394}
1395
1396#[cfg(test)]
1397mod flush_barrier_tests {
1398 use super::*;
1399
1400 #[test]
1401 fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1402 let error = ObjectLogError::Sequencer("manifest failed".into());
1403 let mut queue = Queue::<()> {
1404 items: VecDeque::new(),
1405 bytes: 0,
1406 bytes_in_use: 1,
1407 shutdown: false,
1408 next_seq: 4,
1409 completed_through: 2,
1410 first_failed: Some((2, error.clone())),
1411 force_flush: true,
1412 flush_waiters: Vec::new(),
1413 last_enqueue: None,
1414 oldest_enqueue: None,
1415 };
1416 let mut receivers = Vec::new();
1417 for barrier in [1, 2, 3] {
1418 let (tx, rx) = oneshot::channel();
1419 queue.flush_waiters.push((barrier, tx));
1420 receivers.push(rx);
1421 }
1422 notify_flush_waiters(&mut queue);
1423 assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1424 assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1425 assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1426 assert!(queue.flush_waiters.is_empty());
1427 }
1428}
1429
1430#[cfg(test)]
1431mod ready_commit_group_tests {
1432 use super::*;
1433 use crate::{ManifestSequencer, MemoryBlobStore};
1434 use async_trait::async_trait;
1435 use std::ops::Range;
1436
1437 fn work(
1438 n: u64,
1439 durability: Durability,
1440 ) -> (
1441 FlushWork<()>,
1442 String,
1443 Vec<Bytes>,
1444 oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1445 ) {
1446 let (tx, rx) = oneshot::channel();
1447 let (mut work, key, chunks) = prepare_flush_work(
1448 "data/",
1449 "",
1450 n,
1451 vec![Pending {
1452 partition: PartitionKey("shared".into()),
1453 record_count: 1,
1454 payload: Bytes::from(format!("body-{n}")),
1455 meta: (),
1456 epoch: 0,
1457 durability,
1458 responder: Some(tx),
1459 seq: n,
1460 }],
1461 );
1462 work.put_result = Some(Ok(Duration::from_millis(n)));
1463 (work, key, chunks, rx)
1464 }
1465
1466 #[test]
1467 fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1468 let mut pending = (1..=5)
1469 .map(|n| work(n, Durability::Sequenced).0)
1470 .collect::<VecDeque<_>>();
1471 let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1472 let mut released = 0;
1473 for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1474 let group = take_ready_commit_group(&mut pending, 2).unwrap();
1475 assert_eq!(
1476 group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1477 expected
1478 );
1479 assert_eq!(group.data_objects, expected.len());
1480 assert_eq!(group.responders.len(), expected.len());
1481 assert_eq!(group.max_seq, *expected.last().unwrap());
1482 for (location, n) in group.locations.iter().zip(&expected) {
1483 assert_eq!(location.object_id, format!("data/{n:020}"));
1484 assert_eq!(location.byte_start, 0);
1485 assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1486 }
1487 released += group.bytes;
1488 }
1489 assert!(pending.is_empty());
1490 assert_eq!(released, original_bytes);
1491
1492 let mut first = work(1, Durability::Sequenced).0;
1493 first.put_result = None;
1494 let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1495 assert!(take_ready_commit_group(&mut pending, 8).is_none());
1496 assert_eq!(
1497 pending.len(),
1498 2,
1499 "a later ready PUT cannot pass an unfinished head"
1500 );
1501 pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1502 let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1503 assert_eq!(failed.data_objects, 1);
1504 assert!(failed.put_result.unwrap().is_err());
1505 assert_eq!(pending[0].max_seq, 2);
1506
1507 let mut failed = work(2, Durability::Sequenced).0;
1508 failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1509 let mut pending = VecDeque::from([
1510 work(1, Durability::Sequenced).0,
1511 failed,
1512 work(3, Durability::Sequenced).0,
1513 ]);
1514 assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1515 assert_eq!(
1516 pending.len(),
1517 2,
1518 "a success group cannot cross a failed PUT"
1519 );
1520 let mut pending = VecDeque::from([
1521 work(1, Durability::Sequenced).0,
1522 work(2, Durability::Sequenced).0,
1523 ]);
1524 assert_eq!(
1525 take_ready_commit_group(&mut pending, 1)
1526 .unwrap()
1527 .data_objects,
1528 1
1529 );
1530 assert_eq!(
1531 pending.len(),
1532 1,
1533 "one-object sequencers retain their call boundary"
1534 );
1535 }
1536
1537 struct GatedManifestStore {
1538 inner: MemoryBlobStore,
1539 entered: std::sync::mpsc::SyncSender<()>,
1540 release: Mutex<Option<oneshot::Receiver<()>>>,
1541 fail: bool,
1542 panic_on_commit: bool,
1543 }
1544
1545 #[async_trait]
1546 impl BlobStore for GatedManifestStore {
1547 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1548 if key.starts_with("manifest/") {
1549 let release = self.release.lock().unwrap().take();
1550 if let Some(release) = release {
1551 self.entered.send(()).unwrap();
1552 release.await.unwrap();
1553 assert!(!self.panic_on_commit, "injected commit panic");
1554 if self.fail {
1555 return Err(ObjectLogError::StorageUnavailable(
1556 "manifest failure".into(),
1557 ));
1558 }
1559 }
1560 }
1561 self.inner.put(key, value).await
1562 }
1563 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1564 self.inner.get(key).await
1565 }
1566 async fn get_range(
1567 &self,
1568 key: &str,
1569 range: Range<u64>,
1570 ) -> Result<Option<Bytes>, ObjectLogError> {
1571 self.inner.get_range(key, range).await
1572 }
1573 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1574 self.inner.list(prefix).await
1575 }
1576 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1577 self.inner.delete(key).await
1578 }
1579 fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1580 self.inner.take_media_op_stats()
1581 }
1582 }
1583
1584 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1585 async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1586 for fail in [false, true] {
1587 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1588 let (release_tx, release_rx) = oneshot::channel();
1589 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1590 inner: MemoryBlobStore::new(),
1591 entered: entered_tx,
1592 release: Mutex::new(Some(release_rx)),
1593 fail,
1594 panic_on_commit: false,
1595 });
1596 let sequencer = Arc::new(
1597 ManifestSequencer::open(blob.clone(), "manifest/")
1598 .await
1599 .unwrap(),
1600 );
1601 assert!(sequencer.supports_multi_object_commit());
1602 let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1603 let (mut second, second_key, second_chunks, mut durable_rx) =
1604 work(2, Durability::Durable);
1605 second.batch[0].partition = PartitionKey("other".into());
1606 let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1607 blob.put_chunks(&first_key, first_chunks).await.unwrap();
1608 blob.put_chunks(&second_key, second_chunks).await.unwrap();
1609 blob.put_chunks(&third_key, third_chunks).await.unwrap();
1610 let mut pending = VecDeque::from([first, second, third]);
1611 let group = take_ready_commit_group(&mut pending, 8).unwrap();
1612 let bytes = group.bytes;
1613 assert_eq!(group.data_objects, 3);
1614 let (flush_tx, mut flush_rx) = oneshot::channel();
1615 let mut config = FlushConfig::default();
1616 config.budget.enabled = false;
1617 let shared: Arc<Shared<()>> = Arc::new(Shared {
1618 queue: Mutex::new(Queue {
1619 items: VecDeque::new(),
1620 bytes: 0,
1621 bytes_in_use: bytes,
1622 shutdown: false,
1623 next_seq: 4,
1624 completed_through: 0,
1625 first_failed: None,
1626 force_flush: true,
1627 flush_waiters: vec![(3, flush_tx)],
1628 last_enqueue: None,
1629 oldest_enqueue: None,
1630 }),
1631 cv: Condvar::new(),
1632 max_buffered_bytes: bytes,
1633 budget: Mutex::new(BudgetRuntime::new(config.budget)),
1634 flush_config: config,
1635 });
1636 let worker = {
1637 let shared = shared.clone();
1638 let blob = blob.clone();
1639 let sequencer = sequencer.clone();
1640 std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1641 };
1642 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1643 assert!(matches!(
1644 sequenced_rx.try_recv(),
1645 Err(oneshot::error::TryRecvError::Empty)
1646 ));
1647 assert!(matches!(
1648 third_rx.try_recv(),
1649 Err(oneshot::error::TryRecvError::Empty)
1650 ));
1651 assert!(matches!(
1652 flush_rx.try_recv(),
1653 Err(oneshot::error::TryRecvError::Empty)
1654 ));
1655 let durable = durable_rx.try_recv().unwrap().unwrap();
1656 assert!(durable.durable && !durable.sequenced);
1657 release_tx.send(()).unwrap();
1658 assert_eq!(worker.join().unwrap(), bytes);
1659 if fail {
1660 assert!(sequenced_rx.await.unwrap().is_err());
1661 assert!(third_rx.await.unwrap().is_err());
1662 assert!(flush_rx.await.unwrap().is_err());
1663 assert_eq!(
1664 sequencer
1665 .high_watermark(&PartitionKey("shared".into()))
1666 .unwrap(),
1667 0
1668 );
1669 assert!(blob.list("manifest/").await.unwrap().is_empty());
1670 } else {
1671 assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1672 assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1673 flush_rx.await.unwrap().unwrap();
1674 assert_eq!(blob.list("manifest/").await.unwrap().len(), 3);
1675 }
1676 assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1677 let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1678 .await
1679 .unwrap();
1680 for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1681 let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1682 assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1683 for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1684 assert_eq!(entry.base_offset, offset as i64);
1685 let loc = &entry.location;
1686 assert_eq!(loc.object_id, format!("data/{object:020}"));
1687 let body = blob
1688 .get_range(
1689 &loc.object_id,
1690 loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1691 )
1692 .await
1693 .unwrap()
1694 .unwrap();
1695 assert_eq!(body, format!("body-{object}"));
1696 }
1697 }
1698 }
1699 }
1700
1701 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1702 async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1703 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1704 let (release_tx, release_rx) = oneshot::channel();
1705 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1706 inner: MemoryBlobStore::new(),
1707 entered: entered_tx,
1708 release: Mutex::new(Some(release_rx)),
1709 fail: false,
1710 panic_on_commit: false,
1711 });
1712 let sequencer = Arc::new(
1713 ManifestSequencer::open(blob.clone(), "manifest/")
1714 .await
1715 .unwrap(),
1716 );
1717 let mut config = FlushConfig::default();
1718 config.max_batches = 1;
1719 config.max_inflight_flushes = 4;
1720 config.linger = Duration::ZERO;
1721 config.budget.enabled = false;
1722 let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1723 let first = {
1724 let engine = engine.clone();
1725 tokio::spawn(async move {
1726 engine
1727 .produce(
1728 PartitionKey("p".into()),
1729 Bytes::from_static(b"first"),
1730 1,
1731 (),
1732 Durability::Sequenced,
1733 )
1734 .await
1735 })
1736 };
1737 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1738 for n in 1..4 {
1739 engine
1740 .produce(
1741 PartitionKey("p".into()),
1742 Bytes::from(format!("later-{n}")),
1743 1,
1744 (),
1745 Durability::Buffered,
1746 )
1747 .await
1748 .unwrap();
1749 }
1750 let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1751 loop {
1752 if blob.list("data/").await.unwrap().len() == 4 {
1753 break;
1754 }
1755 tokio::time::sleep(Duration::from_millis(1)).await;
1756 }
1757 })
1758 .await
1759 .is_ok();
1760 let early_ack = first.is_finished();
1761 release_tx.send(()).unwrap();
1763 first.await.unwrap().unwrap();
1764 engine.flush().await.unwrap();
1765 let rows = engine
1766 .fetch(&PartitionKey("p".into()), 0, 1024)
1767 .await
1768 .unwrap();
1769 assert_eq!(
1770 rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
1771 vec![0, 1, 2, 3]
1772 );
1773 drop(engine);
1774 assert!(
1775 !early_ack,
1776 "sequenced acknowledgement preceded manifest durability"
1777 );
1778 assert!(
1779 upload_progress,
1780 "blocked manifest prevented independent data uploads"
1781 );
1782 }
1783
1784 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1785 async fn commit_completion_releases_admission_bytes_without_more_uploads() {
1786 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1787 let sequencer = Arc::new(
1788 ManifestSequencer::open(blob.clone(), "manifest/")
1789 .await
1790 .unwrap(),
1791 );
1792 let mut config = FlushConfig::default();
1793 config.max_bytes = 4;
1794 config.max_buffered_bytes = 4;
1795 config.max_batches = 1;
1796 config.max_inflight_flushes = 4;
1797 config.linger = Duration::ZERO;
1798 config.budget.enabled = false;
1799 let engine = LogEngine::new(blob, sequencer, config, "data/");
1800 engine
1801 .produce(
1802 PartitionKey("p".into()),
1803 Bytes::from_static(b"full"),
1804 1,
1805 (),
1806 Durability::Sequenced,
1807 )
1808 .await
1809 .unwrap();
1810 tokio::time::timeout(Duration::from_secs(5), async {
1811 while engine.buffer_stats().bytes_in_use != 0 {
1812 tokio::time::sleep(Duration::from_millis(1)).await;
1813 }
1814 })
1815 .await
1816 .expect("completed committer stranded admission bytes");
1817 engine
1818 .produce(
1819 PartitionKey("p".into()),
1820 Bytes::from_static(b"next"),
1821 1,
1822 (),
1823 Durability::Sequenced,
1824 )
1825 .await
1826 .unwrap();
1827 engine.flush().await.unwrap();
1828 assert_eq!(
1829 engine
1830 .fetch(&PartitionKey("p".into()), 0, 100)
1831 .await
1832 .unwrap()
1833 .len(),
1834 2
1835 );
1836 }
1837
1838 struct FailureInjectingStore {
1839 inner: Arc<dyn BlobStore>,
1840 second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
1841 second_upload_completed: std::sync::atomic::AtomicBool,
1842 fail_data_listing: std::sync::atomic::AtomicBool,
1843 }
1844
1845 impl FailureInjectingStore {
1846 fn new(inner: Arc<dyn BlobStore>) -> Self {
1847 Self {
1848 inner,
1849 second_upload_gate: Mutex::new(None),
1850 second_upload_completed: std::sync::atomic::AtomicBool::new(false),
1851 fail_data_listing: std::sync::atomic::AtomicBool::new(false),
1852 }
1853 }
1854 }
1855
1856 #[async_trait]
1857 impl BlobStore for FailureInjectingStore {
1858 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1859 let gate = if key == "data/00000000000000000002" {
1860 self.second_upload_gate.lock().unwrap().take()
1861 } else {
1862 None
1863 };
1864 if let Some((entered, release)) = gate {
1865 entered.send(()).unwrap();
1866 release.await.unwrap();
1867 let result = self.inner.put(key, value).await;
1868 if result.is_ok() {
1869 self.second_upload_completed
1870 .store(true, std::sync::atomic::Ordering::SeqCst);
1871 }
1872 return result;
1873 }
1874 self.inner.put(key, value).await
1875 }
1876 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1877 self.inner.get(key).await
1878 }
1879 async fn get_range(
1880 &self,
1881 key: &str,
1882 range: Range<u64>,
1883 ) -> Result<Option<Bytes>, ObjectLogError> {
1884 self.inner.get_range(key, range).await
1885 }
1886 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1887 if prefix == "data/"
1888 && self
1889 .fail_data_listing
1890 .load(std::sync::atomic::Ordering::SeqCst)
1891 {
1892 return Err(ObjectLogError::StorageUnavailable(
1893 "injected object listing failure".into(),
1894 ));
1895 }
1896 self.inner.list(prefix).await
1897 }
1898 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1899 self.inner.delete(key).await
1900 }
1901 }
1902
1903 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1904 async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
1905 let inner = Arc::new(MemoryBlobStore::new());
1906 let key = "data/00000000000000000001";
1907 let original = Bytes::from_static(b"existing immutable object");
1908 inner.put(key, original.clone()).await.unwrap();
1909 let store = Arc::new(FailureInjectingStore::new(inner.clone()));
1910 store
1911 .fail_data_listing
1912 .store(true, std::sync::atomic::Ordering::SeqCst);
1913 let mut config = FlushConfig::default();
1914 config.max_batches = 1;
1915 config.max_inflight_flushes = 4;
1916 config.linger = Duration::ZERO;
1917 config.budget.enabled = false;
1918 let engine = LogEngine::new(
1919 store,
1920 Arc::new(crate::InMemorySequencer::new()),
1921 config,
1922 "data/",
1923 );
1924 let result = tokio::time::timeout(
1925 Duration::from_secs(5),
1926 engine.produce(
1927 PartitionKey("p".into()),
1928 Bytes::from_static(b"must not overwrite"),
1929 1,
1930 (),
1931 Durability::Sequenced,
1932 ),
1933 )
1934 .await;
1935 let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
1936 tokio::task::spawn_blocking(move || drop(engine))
1937 .await
1938 .unwrap();
1939 let retained = inner.get(key).await.unwrap();
1940 assert_eq!(
1941 retained,
1942 Some(original),
1943 "failed listing must preserve sealed bytes"
1944 );
1945 assert!(
1946 result.unwrap().is_err(),
1947 "failed prefix recovery must reject admission"
1948 );
1949 assert!(
1950 flushed.unwrap().is_err(),
1951 "flush must report failed initialization"
1952 );
1953 assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
1954 }
1955
1956 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1957 async fn engines_commit_and_close_independently() {
1958 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1959 let (release_tx, release_rx) = oneshot::channel();
1960 let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1961 inner: MemoryBlobStore::new(),
1962 entered: entered_tx,
1963 release: Mutex::new(Some(release_rx)),
1964 fail: false,
1965 panic_on_commit: false,
1966 })));
1967 let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
1968 let mut config = FlushConfig::default();
1969 config.max_batches = 1;
1970 config.max_inflight_flushes = 4;
1971 config.linger = Duration::ZERO;
1972 config.budget.enabled = false;
1973 let a_seq = Arc::new(
1974 ManifestSequencer::open(a_blob.clone(), "manifest/")
1975 .await
1976 .unwrap(),
1977 );
1978 let b_seq = Arc::new(
1979 ManifestSequencer::open(b_blob.clone(), "manifest/")
1980 .await
1981 .unwrap(),
1982 );
1983 let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
1984 a.produce(
1985 PartitionKey("p".into()),
1986 Bytes::from_static(b"a-first"),
1987 1,
1988 (),
1989 Durability::Buffered,
1990 )
1991 .await
1992 .unwrap();
1993 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1994 let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
1995 let b_result = tokio::time::timeout(
1996 Duration::from_secs(5),
1997 b.produce(
1998 PartitionKey("p".into()),
1999 Bytes::from_static(b"b-first"),
2000 1,
2001 (),
2002 Durability::Sequenced,
2003 ),
2004 )
2005 .await;
2006 let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
2007 let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
2008 release_tx.send(()).unwrap();
2010 if b_closed.is_err() {
2011 closing_b.await.unwrap();
2012 }
2013 assert!(
2014 b_closed.is_ok(),
2015 "closing B waited for A's blocked manifest"
2016 );
2017 assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
2018 a.flush().await.unwrap();
2019 assert_eq!(
2020 a.produce(
2021 PartitionKey("p".into()),
2022 Bytes::from_static(b"a-second"),
2023 1,
2024 (),
2025 Durability::Sequenced
2026 )
2027 .await
2028 .unwrap()
2029 .base_offset,
2030 Some(1)
2031 );
2032 let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
2033 assert_eq!(
2034 rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
2035 vec![
2036 Bytes::from_static(b"a-first"),
2037 Bytes::from_static(b"a-second")
2038 ]
2039 );
2040 tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
2041 }
2042
2043 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2044 async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
2045 let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
2046 let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
2047 let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
2048 let (upload_release_tx, upload_release_rx) = oneshot::channel();
2049 let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
2050 inner: MemoryBlobStore::new(),
2051 entered: manifest_entered_tx,
2052 release: Mutex::new(Some(manifest_release_rx)),
2053 fail: false,
2054 panic_on_commit: true,
2055 })));
2056 *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
2057 let mut config = FlushConfig::default();
2058 config.max_batches = 1;
2059 config.max_inflight_flushes = 4;
2060 config.linger = Duration::ZERO;
2061 config.budget.enabled = false;
2062 let sequencer = Arc::new(
2063 ManifestSequencer::open(store.clone(), "manifest/")
2064 .await
2065 .unwrap(),
2066 );
2067 let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
2068 engine
2069 .produce(
2070 PartitionKey("p".into()),
2071 Bytes::from_static(b"first"),
2072 1,
2073 (),
2074 Durability::Buffered,
2075 )
2076 .await
2077 .unwrap();
2078 manifest_entered_rx
2079 .recv_timeout(Duration::from_secs(5))
2080 .unwrap();
2081 engine
2082 .produce(
2083 PartitionKey("p".into()),
2084 Bytes::from_static(b"second"),
2085 1,
2086 (),
2087 Durability::Buffered,
2088 )
2089 .await
2090 .unwrap();
2091 upload_entered_rx
2092 .recv_timeout(Duration::from_secs(5))
2093 .unwrap();
2094 manifest_release_tx.send(()).unwrap();
2095 tokio::time::timeout(Duration::from_secs(5), async {
2096 while !engine.shared.queue.lock().unwrap().shutdown {
2097 tokio::time::sleep(Duration::from_millis(1)).await;
2098 }
2099 })
2100 .await
2101 .unwrap();
2102 let flush_failed = engine.flush().await.is_err();
2103 let upload_was_owned = upload_release_tx.send(()).is_ok();
2106 tokio::time::timeout(
2107 Duration::from_secs(5),
2108 tokio::task::spawn_blocking(move || drop(engine)),
2109 )
2110 .await
2111 .unwrap()
2112 .unwrap();
2113 assert!(flush_failed);
2114 assert!(upload_was_owned);
2115 assert!(
2116 store
2117 .second_upload_completed
2118 .load(std::sync::atomic::Ordering::SeqCst)
2119 );
2120 assert_eq!(store.list("data/").await.unwrap().len(), 2);
2121 let sequencer = Arc::new(
2122 ManifestSequencer::open(store.clone(), "manifest/")
2123 .await
2124 .unwrap(),
2125 );
2126 let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2127 assert!(
2128 reopened
2129 .fetch(&PartitionKey("p".into()), 0, 100)
2130 .await
2131 .unwrap()
2132 .is_empty()
2133 );
2134 assert_eq!(
2135 reopened
2136 .produce(
2137 PartitionKey("p".into()),
2138 Bytes::from_static(b"after-reopen"),
2139 1,
2140 (),
2141 Durability::Sequenced
2142 )
2143 .await
2144 .unwrap()
2145 .base_offset,
2146 Some(0)
2147 );
2148 let rows = reopened
2149 .fetch(&PartitionKey("p".into()), 0, 100)
2150 .await
2151 .unwrap();
2152 assert_eq!(rows.len(), 1);
2153 assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2154 assert_eq!(store.list("data/").await.unwrap().len(), 3);
2155 tokio::task::spawn_blocking(move || drop(reopened))
2156 .await
2157 .unwrap();
2158 }
2159
2160 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2161 async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2162 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2163 let (release_tx, release_rx) = oneshot::channel();
2164 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2165 inner: MemoryBlobStore::new(),
2166 entered: entered_tx,
2167 release: Mutex::new(Some(release_rx)),
2168 fail: false,
2169 panic_on_commit: false,
2170 });
2171 let sequencer = Arc::new(
2172 ManifestSequencer::open(blob.clone(), "manifest/")
2173 .await
2174 .unwrap(),
2175 );
2176 let mut config = FlushConfig::default();
2177 config.max_batches = 1;
2178 config.max_inflight_flushes = 4;
2179 config.linger = Duration::ZERO;
2180 config.budget.enabled = false;
2181 let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2182 engine
2183 .produce(
2184 PartitionKey("p".into()),
2185 Bytes::from_static(b"first"),
2186 1,
2187 (),
2188 Durability::Buffered,
2189 )
2190 .await
2191 .unwrap();
2192 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2193 engine
2194 .produce(
2195 PartitionKey("p".into()),
2196 Bytes::from_static(b"second"),
2197 1,
2198 (),
2199 Durability::Buffered,
2200 )
2201 .await
2202 .unwrap();
2203 let shared = engine.shared.clone();
2204 let dropping = tokio::task::spawn_blocking(move || drop(engine));
2205 tokio::time::timeout(Duration::from_secs(5), async {
2206 while !shared.queue.lock().unwrap().shutdown {
2207 tokio::time::sleep(Duration::from_millis(1)).await;
2208 }
2209 })
2210 .await
2211 .unwrap();
2212 let premature = dropping.is_finished();
2213 release_tx.send(()).unwrap();
2214 tokio::time::timeout(Duration::from_secs(5), dropping)
2215 .await
2216 .unwrap()
2217 .unwrap();
2218 assert!(!premature, "shutdown returned before manifest durability");
2219 let reopened = Arc::new(
2220 ManifestSequencer::open(blob.clone(), "manifest/")
2221 .await
2222 .unwrap(),
2223 );
2224 let engine = LogEngine::new(blob, reopened, config, "data/");
2225 let rows = engine
2226 .fetch(&PartitionKey("p".into()), 0, 100)
2227 .await
2228 .unwrap();
2229 assert_eq!(
2230 rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2231 vec![b"first".as_slice(), b"second".as_slice()]
2232 );
2233 }
2234
2235 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2236 async fn panicked_committer_closes_admission_and_fails_flush() {
2237 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2238 let (release_tx, release_rx) = oneshot::channel();
2239 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2240 inner: MemoryBlobStore::new(),
2241 entered: entered_tx,
2242 release: Mutex::new(Some(release_rx)),
2243 fail: false,
2244 panic_on_commit: true,
2245 });
2246 let sequencer = Arc::new(
2247 ManifestSequencer::open(blob.clone(), "manifest/")
2248 .await
2249 .unwrap(),
2250 );
2251 let mut config = FlushConfig::default();
2252 config.max_batches = 1;
2253 config.max_inflight_flushes = 4;
2254 config.linger = Duration::ZERO;
2255 config.budget.enabled = false;
2256 let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2257 let first = {
2258 let engine = engine.clone();
2259 tokio::spawn(async move {
2260 engine
2261 .produce(
2262 PartitionKey("p".into()),
2263 Bytes::from_static(b"first"),
2264 1,
2265 (),
2266 Durability::Sequenced,
2267 )
2268 .await
2269 })
2270 };
2271 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2272 engine
2273 .produce(
2274 PartitionKey("p".into()),
2275 Bytes::from_static(b"later"),
2276 1,
2277 (),
2278 Durability::Buffered,
2279 )
2280 .await
2281 .unwrap();
2282 let flushing = {
2283 let engine = engine.clone();
2284 tokio::spawn(async move { engine.flush().await })
2285 };
2286 release_tx.send(()).unwrap();
2287 assert!(
2288 tokio::time::timeout(Duration::from_secs(5), first)
2289 .await
2290 .unwrap()
2291 .unwrap()
2292 .is_err()
2293 );
2294 assert!(
2295 tokio::time::timeout(Duration::from_secs(5), flushing)
2296 .await
2297 .unwrap()
2298 .unwrap()
2299 .is_err()
2300 );
2301 tokio::time::timeout(Duration::from_secs(5), async {
2302 while !engine.shared.queue.lock().unwrap().shutdown {
2303 tokio::time::sleep(Duration::from_millis(1)).await;
2304 }
2305 })
2306 .await
2307 .unwrap();
2308 assert!(
2309 engine
2310 .produce(
2311 PartitionKey("p".into()),
2312 Bytes::from_static(b"rejected"),
2313 1,
2314 (),
2315 Durability::Buffered
2316 )
2317 .await
2318 .is_err()
2319 );
2320 assert!(engine.flush().await.is_err());
2321 }
2322 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2323 async fn committed_reads_continue_during_new_manifest_publication() {
2324 for fail in [false, true] {
2325 let memory = MemoryBlobStore::new();
2326 let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2327 let seed_sequencer = Arc::new(
2328 ManifestSequencer::open(seed_blob.clone(), "manifest/")
2329 .await
2330 .unwrap(),
2331 );
2332 let mut config = FlushConfig::default();
2333 config.max_batches = 1;
2334 config.max_inflight_flushes = 4;
2335 config.linger = Duration::ZERO;
2336 config.budget.enabled = false;
2337 let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2338 seed.produce(
2339 PartitionKey("p".into()),
2340 Bytes::from_static(b"committed"),
2341 1,
2342 (),
2343 Durability::Sequenced,
2344 )
2345 .await
2346 .unwrap();
2347 drop(seed);
2348 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2349 let (release_tx, release_rx) = oneshot::channel();
2350 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2351 inner: memory,
2352 entered: entered_tx,
2353 release: Mutex::new(Some(release_rx)),
2354 fail,
2355 panic_on_commit: false,
2356 });
2357 let sequencer = Arc::new(
2358 ManifestSequencer::open(blob.clone(), "manifest/")
2359 .await
2360 .unwrap(),
2361 );
2362 let engine = Arc::new(LogEngine::new(
2363 blob.clone(),
2364 sequencer.clone(),
2365 config,
2366 "data/",
2367 ));
2368 let producing = {
2369 let engine = engine.clone();
2370 tokio::spawn(async move {
2371 engine
2372 .produce(
2373 PartitionKey("p".into()),
2374 Bytes::from_static(b"new"),
2375 1,
2376 (),
2377 Durability::Sequenced,
2378 )
2379 .await
2380 })
2381 };
2382 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2383 let mut reading = tokio::task::spawn_blocking(move || {
2384 let p = PartitionKey("p".into());
2385 (
2386 sequencer.high_watermark(&p).unwrap(),
2387 sequencer.lookup(&p, 0).unwrap(),
2388 sequencer.snapshot(),
2389 )
2390 });
2391 let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2392 let early_ack = producing.is_finished();
2393 release_tx.send(()).unwrap();
2395 let outcome = producing.await.unwrap();
2396 assert_eq!(outcome.is_err(), fail);
2397 let reads_progressed = observed.is_ok();
2398 let (watermark, entries, snapshot) = match observed {
2399 Ok(result) => result.unwrap(),
2400 Err(_) => reading.await.unwrap(),
2401 };
2402 assert!(!early_ack);
2403 assert!(
2404 reads_progressed,
2405 "durable manifest publication blocked committed index reads"
2406 );
2407 assert_eq!(watermark, 1);
2408 assert_eq!(entries.len(), 1);
2409 assert_eq!(snapshot.manifest_count, 1);
2410 let rows = engine
2411 .fetch(&PartitionKey("p".into()), 0, 100)
2412 .await
2413 .unwrap();
2414 assert_eq!(rows.len(), if fail { 1 } else { 2 });
2415 assert_eq!(rows[0].payload.as_ref(), b"committed");
2416 drop(engine);
2417 let reopened = Arc::new(
2418 ManifestSequencer::open(blob.clone(), "manifest/")
2419 .await
2420 .unwrap(),
2421 );
2422 let engine = LogEngine::new(blob, reopened, config, "data/");
2423 assert_eq!(
2424 engine
2425 .fetch(&PartitionKey("p".into()), 0, 100)
2426 .await
2427 .unwrap()
2428 .len(),
2429 if fail { 1 } else { 2 }
2430 );
2431 }
2432 }
2433 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2434 async fn manifest_mutations_remain_serialized_while_readers_progress() {
2435 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2436 let (release_tx, release_rx) = oneshot::channel();
2437 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2438 inner: MemoryBlobStore::new(),
2439 entered: entered_tx,
2440 release: Mutex::new(Some(release_rx)),
2441 fail: false,
2442 panic_on_commit: false,
2443 });
2444 let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2445 let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2446 tokio::task::spawn_blocking(move || {
2447 sequencer.commit(&[CommitBatch {
2448 partition: PartitionKey("p".into()),
2449 record_count: 1,
2450 location: BatchLocation {
2451 object_id: key.into(),
2452 byte_start: 0,
2453 byte_len: 1,
2454 },
2455 meta: &(),
2456 epoch: 0,
2457 }])
2458 })
2459 };
2460 let first = commit(sequencer.clone(), "data/first");
2461 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2462 let second = commit(sequencer.clone(), "data/second");
2463 let truncating = {
2464 let sequencer = sequencer.clone();
2465 tokio::task::spawn_blocking(move || {
2466 sequencer.truncate_before(&PartitionKey("p".into()), 1)
2467 })
2468 };
2469 let reading = {
2470 let sequencer = sequencer.clone();
2471 tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2472 };
2473 let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2474 release_tx.send(()).unwrap();
2475 assert_eq!(
2476 observed.expect("committed reads blocked").unwrap().unwrap(),
2477 0
2478 );
2479 let first = tokio::time::timeout(Duration::from_secs(5), first)
2480 .await
2481 .unwrap()
2482 .unwrap()
2483 .unwrap();
2484 let second = tokio::time::timeout(Duration::from_secs(5), second)
2485 .await
2486 .unwrap()
2487 .unwrap()
2488 .unwrap();
2489 let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2490 .await
2491 .unwrap()
2492 .unwrap()
2493 .unwrap();
2494 assert_eq!(
2495 first,
2496 vec![CommitOutcome::Assigned {
2497 base_offset: 0,
2498 record_count: 1
2499 }]
2500 );
2501 assert_eq!(
2502 second,
2503 vec![CommitOutcome::Assigned {
2504 base_offset: 1,
2505 record_count: 1
2506 }]
2507 );
2508 assert_eq!(dropped, vec!["data/first".to_string()]);
2509 let p = PartitionKey("p".into());
2510 assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2511 assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2512 let entries = sequencer.lookup(&p, 0).unwrap();
2513 assert_eq!(entries.len(), 1);
2514 assert_eq!(entries[0].location.object_id, "data/second");
2515 assert_eq!(sequencer.snapshot().manifest_count, 2);
2516 }
2517}
2518
2519#[cfg(test)]
2520mod idle_batch_wait_tests {
2521 use super::*;
2522 use std::sync::mpsc;
2523
2524 fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2525 Arc::new(Shared {
2526 queue: Mutex::new(Queue {
2527 items: VecDeque::new(),
2528 bytes: 0,
2529 bytes_in_use: 0,
2530 shutdown: false,
2531 next_seq: 1,
2532 completed_through: 0,
2533 first_failed: None,
2534 force_flush: false,
2535 flush_waiters: Vec::new(),
2536 last_enqueue: None,
2537 oldest_enqueue: None,
2538 }),
2539 cv: Condvar::new(),
2540 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2541 budget: Mutex::new(BudgetRuntime::new(config.budget)),
2542 flush_config: config,
2543 })
2544 }
2545
2546 #[test]
2547 fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2548 for linger in [Duration::ZERO, Duration::from_millis(1)] {
2549 for shutdown in [false, true] {
2550 let config = FlushConfig {
2551 linger,
2552 ..FlushConfig::default()
2553 };
2554 let shared = empty_shared(config);
2555 let (started_tx, started_rx) = mpsc::channel();
2556 let (result_tx, result_rx) = mpsc::channel();
2557 let worker_shared = shared.clone();
2558 let worker = std::thread::spawn(move || {
2559 started_tx.send(()).unwrap();
2560 assert!(
2561 result_tx
2562 .send(take_batch(&worker_shared, config, true))
2563 .is_ok()
2564 );
2565 });
2566 started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2567 let early = result_rx.recv_timeout(Duration::from_millis(100));
2568 let returned_while_idle = early.is_ok();
2569 {
2570 let mut q = shared.queue.lock().unwrap();
2571 if shutdown {
2572 q.shutdown = true;
2573 } else {
2574 let payload = Bytes::from_static(b"first-after-idle");
2575 q.bytes = payload.len();
2576 q.bytes_in_use = payload.len();
2577 q.next_seq = 2;
2578 q.last_enqueue = Some(Instant::now());
2579 q.oldest_enqueue = q.last_enqueue;
2580 q.force_flush = true;
2581 q.items.push_back(Pending {
2582 partition: PartitionKey("p".into()),
2583 record_count: 1,
2584 payload,
2585 meta: (),
2586 epoch: 0,
2587 durability: Durability::Buffered,
2588 responder: None,
2589 seq: 1,
2590 });
2591 }
2592 shared.cv.notify_all();
2593 }
2594 let result = early
2595 .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2596 .unwrap();
2597 worker.join().unwrap();
2598 assert!(
2599 !returned_while_idle,
2600 "idle worker polled with linger={linger:?}"
2601 );
2602 match result {
2603 TakeBatch::Shutdown => assert!(shutdown),
2604 TakeBatch::Batch(items) => {
2605 assert!(!shutdown);
2606 assert_eq!(items.len(), 1);
2607 assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2608 assert_eq!(items[0].seq, 1);
2609 }
2610 TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2611 }
2612 }
2613 }
2614 }
2615}