1use crate::budget::{
4 BudgetConfig, BudgetMode, BudgetRuntime, EffectiveKnob, EffectiveReason, PipelineSnapshot,
5};
6use crate::sequencer::BatchLocation;
7use crate::{
8 BlobStore, CommitBatch, CommitOutcome, IndexEntry, ObjectLogError, PartitionKey, Sequencer,
9};
10use bytes::Bytes;
11use std::collections::{HashSet, VecDeque};
12use std::sync::{Arc, Condvar, Mutex};
13use std::thread::JoinHandle;
14use std::time::{Duration, Instant};
15use tokio::sync::oneshot;
16use tokio::task::JoinHandle as TokioJoinHandle;
17
18const STORAGE_RETRY_ATTEMPTS: usize = 5;
19const STORAGE_RETRY_BASE_DELAY: Duration = Duration::from_millis(25);
20
21#[derive(Clone, Copy, Debug, PartialEq, Eq)]
23pub enum Durability {
24 Buffered,
27 Durable,
30 Sequenced,
33}
34
35#[derive(Clone, Copy, Debug)]
37pub struct FlushConfig {
38 pub max_bytes: usize,
43 pub max_batches: usize,
45 pub linger: Duration,
51 pub max_inflight_flushes: usize,
55 pub max_buffered_bytes: usize,
58 pub budget: BudgetConfig,
60}
61
62impl Default for FlushConfig {
63 fn default() -> Self {
64 Self {
65 max_bytes: 1024 * 1024 * 1024,
67 max_batches: 10_000,
68 linger: Duration::from_millis(50),
69 max_inflight_flushes: 1,
71 max_buffered_bytes: 2 * 1024 * 1024 * 1024,
72 budget: BudgetConfig::default(),
73 }
74 }
75}
76
77#[derive(Clone, Debug, PartialEq, Eq)]
79pub struct AppendOutcome {
80 pub base_offset: Option<i64>,
82 pub last_offset: Option<i64>,
84 pub durable: bool,
86 pub sequenced: bool,
88}
89
90#[derive(Clone, Debug, PartialEq, Eq)]
92pub struct FetchedBatch {
93 pub base_offset: i64,
95 pub record_count: i32,
97 pub payload: Bytes,
99}
100
101#[derive(Clone, Debug, PartialEq, Eq)]
103pub struct BufferStats {
104 pub queued_bytes: usize,
106 pub bytes_in_use: usize,
108 pub queued_batches: usize,
110 pub max_buffered_bytes: usize,
112}
113
114type Responder = oneshot::Sender<Result<AppendOutcome, ObjectLogError>>;
115
116fn retryable_storage_error(err: &ObjectLogError) -> bool {
117 matches!(err, ObjectLogError::StorageUnavailable(_))
118}
119
120async fn retry_delay(attempt: usize) {
121 let multiplier = 1u32 << attempt.min(5);
122 tokio::time::sleep(STORAGE_RETRY_BASE_DELAY * multiplier).await;
123}
124
125async fn put_chunks_with_retries(
126 blob: &Arc<dyn BlobStore>,
127 key: &str,
128 chunks: Vec<Bytes>,
129) -> Result<(), ObjectLogError> {
130 let mut attempt = 0usize;
132 loop {
133 match blob.put_chunks(key, chunks.clone()).await {
134 Ok(()) => return Ok(()),
135 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
136 retry_delay(attempt).await;
137 attempt += 1;
138 }
139 Err(err) => return Err(err),
140 }
141 }
142}
143
144async fn get_range_with_retries(
145 blob: &Arc<dyn BlobStore>,
146 key: &str,
147 range: std::ops::Range<u64>,
148) -> Result<Option<Bytes>, ObjectLogError> {
149 let mut attempt = 0usize;
150 loop {
151 match blob.get_range(key, range.clone()).await {
152 Ok(bytes) => return Ok(bytes),
153 Err(err) if retryable_storage_error(&err) && attempt < STORAGE_RETRY_ATTEMPTS => {
154 retry_delay(attempt).await;
155 attempt += 1;
156 }
157 Err(err) => return Err(err),
158 }
159 }
160}
161
162struct Pending<M> {
163 partition: PartitionKey,
164 record_count: i32,
165 payload: Bytes,
166 meta: M,
167 durability: Durability,
168 responder: Option<Responder>,
169 seq: u64,
171}
172
173struct FlushWork<M> {
174 batch: Vec<Pending<M>>,
175 locations: Vec<BatchLocation>,
176 responders: Vec<(Durability, Option<Responder>)>,
177 bytes: usize,
178 max_seq: u64,
180 data_objects: usize,
182 put: Option<TokioJoinHandle<Result<(), ObjectLogError>>>,
183 put_started: Instant,
184 put_result: Option<Result<Duration, ObjectLogError>>,
185}
186
187struct CommitJob {
188 task: TokioJoinHandle<usize>,
189 first_seq: u64,
190}
191
192enum TakeBatch<M> {
193 Batch(Vec<Pending<M>>),
194 Empty,
195 Shutdown,
196}
197
198type FlushWaiter = (u64, oneshot::Sender<Result<(), ObjectLogError>>);
199
200struct Queue<M> {
201 items: VecDeque<Pending<M>>,
202 bytes: usize,
203 bytes_in_use: usize,
204 shutdown: bool,
205 next_seq: u64,
207 completed_through: u64,
209 first_failed: Option<(u64, ObjectLogError)>,
211 force_flush: bool,
213 flush_waiters: Vec<FlushWaiter>,
215 last_enqueue: Option<Instant>,
217 oldest_enqueue: Option<Instant>,
219}
220
221struct Shared<M> {
222 queue: Mutex<Queue<M>>,
223 cv: Condvar,
224 max_buffered_bytes: usize,
225 budget: Mutex<BudgetRuntime>,
227 flush_config: FlushConfig,
229}
230
231pub struct LogEngine<S: Sequencer> {
240 shared: Arc<Shared<S::Meta>>,
241 blob: Arc<dyn BlobStore>,
242 sequencer: Arc<S>,
243 key_prefix: String,
245 flush_thread: Option<JoinHandle<()>>,
246}
247
248impl<S> LogEngine<S>
249where
250 S: Sequencer + 'static,
251 S::Meta: Send + 'static,
252{
253 pub fn new(
257 blob: Arc<dyn BlobStore>,
258 sequencer: Arc<S>,
259 config: FlushConfig,
260 key_prefix: impl Into<String>,
261 ) -> Self {
262 if let Err(msg) = config.budget.validate() {
263 panic!("invalid FlushConfig.budget: {msg}");
264 }
265 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
266 eprintln!("object-log flush config: {config:?}");
267 }
268 let budget_rt = BudgetRuntime::new(config.budget);
269 let shared = Arc::new(Shared {
270 queue: Mutex::new(Queue {
271 items: VecDeque::new(),
272 bytes: 0,
273 bytes_in_use: 0,
274 shutdown: false,
275 next_seq: 1, completed_through: 0,
277 first_failed: None,
278 force_flush: false,
279 flush_waiters: Vec::new(),
280 last_enqueue: None,
281 oldest_enqueue: None,
282 }),
283 cv: Condvar::new(),
284 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
285 budget: Mutex::new(budget_rt),
286 flush_config: config,
287 });
288 let key_prefix = key_prefix.into();
289 let flush_thread = {
290 let shared = Arc::clone(&shared);
291 let blob = Arc::clone(&blob);
292 let sequencer = Arc::clone(&sequencer);
293 let prefix = key_prefix.clone();
294 std::thread::Builder::new()
295 .name("object-log-flush".into())
296 .spawn(move || flush_loop(shared, blob, sequencer, config, prefix))
297 .expect("spawn flush thread")
298 };
299 Self {
300 shared,
301 blob,
302 sequencer,
303 key_prefix,
304 flush_thread: Some(flush_thread),
305 }
306 }
307
308 pub fn data_prefix(&self) -> &str {
310 &self.key_prefix
311 }
312
313 pub fn blob_store(&self) -> &Arc<dyn BlobStore> {
315 &self.blob
316 }
317
318 pub fn sequencer(&self) -> &Arc<S> {
320 &self.sequencer
321 }
322
323 pub async fn produce(
325 &self,
326 partition: PartitionKey,
327 payload: Bytes,
328 record_count: i32,
329 meta: S::Meta,
330 durability: Durability,
331 ) -> Result<AppendOutcome, ObjectLogError> {
332 if record_count <= 0 {
333 return Err(ObjectLogError::InvalidBatch(
334 "record_count must be > 0".into(),
335 ));
336 }
337 if matches!(durability, Durability::Buffered) {
338 self.enqueue(Pending {
339 partition,
340 record_count,
341 payload,
342 meta,
343 durability,
344 responder: None,
345 seq: 0, })?;
347 return Ok(AppendOutcome {
348 base_offset: None,
349 last_offset: None,
350 durable: false,
351 sequenced: false,
352 });
353 }
354 {
356 let mut budget = self.shared.budget.lock().expect("poisoned");
357 if budget.config.enabled
358 && budget.config.mode == BudgetMode::FailClosed
359 && !budget.reserve_for_fail_closed(Instant::now())
360 {
361 return Err(ObjectLogError::BudgetExceeded(
362 "insufficient durable-ops budget to admit produce".into(),
363 ));
364 }
365 }
366 if self.shared.flush_config.budget.enabled
368 && self.shared.flush_config.budget.mode == BudgetMode::BudgetPriority
369 {
370 let timeout = self.shared.flush_config.budget.admission_timeout;
371 let deadline = Instant::now() + timeout;
372 loop {
373 let mut budget = self.shared.budget.lock().expect("poisoned");
374 if budget.can_admit_now(Instant::now()) {
375 break;
376 }
377 drop(budget);
378 if Instant::now() >= deadline {
379 return Err(ObjectLogError::BudgetExceeded(
380 "timed out waiting for durable-ops budget".into(),
381 ));
382 }
383 std::thread::sleep(Duration::from_millis(1));
384 }
385 }
386 let (tx, rx) = oneshot::channel();
387 self.enqueue(Pending {
388 partition,
389 record_count,
390 payload,
391 meta,
392 durability,
393 responder: Some(tx),
394 seq: 0,
395 })?;
396 rx.await
397 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
398 }
399
400 pub async fn flush(&self) -> Result<(), ObjectLogError> {
409 let rx = {
410 let mut q = self.shared.queue.lock().expect("poisoned");
411 if q.shutdown {
412 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
413 }
414 if q.next_seq == 1 {
415 return Ok(());
417 }
418 let barrier = q.next_seq - 1;
419 if let Some((sequence, error)) = &q.first_failed {
420 if *sequence <= barrier {
421 return Err(error.clone());
422 }
423 }
424 if q.completed_through >= barrier && q.items.is_empty() && q.bytes_in_use == 0 {
425 return Ok(());
426 }
427 let (tx, rx) = oneshot::channel();
428 q.force_flush = true;
429 q.flush_waiters.push((barrier, tx));
430 self.shared.cv.notify_all();
431 rx
432 };
433 rx.await
434 .map_err(|_| ObjectLogError::Sequencer("flush worker stopped".into()))?
435 }
436
437 fn enqueue(&self, mut item: Pending<S::Meta>) -> Result<(), ObjectLogError> {
438 let item_len = item.payload.len();
439 let max_buffered_bytes = self.shared.max_buffered_bytes;
440 let shared = Arc::clone(&self.shared);
441 let mut q = shared.queue.lock().expect("poisoned");
442 while !q.shutdown
443 && q.bytes_in_use > 0
444 && q.bytes_in_use.saturating_add(item_len) > max_buffered_bytes
445 {
446 q = shared.cv.wait(q).expect("poisoned");
447 }
448 if q.shutdown {
449 return Err(ObjectLogError::Sequencer("engine shutting down".into()));
450 }
451 let now = Instant::now();
452 item.seq = q.next_seq;
453 q.next_seq = q.next_seq.saturating_add(1);
454 q.bytes += item_len;
455 q.bytes_in_use += item_len;
456 q.last_enqueue = Some(now);
457 if q.oldest_enqueue.is_none() {
458 q.oldest_enqueue = Some(now);
459 }
460 q.items.push_back(item);
461 shared.cv.notify_all();
462 Ok(())
463 }
464
465 pub async fn fetch(
467 &self,
468 partition: &PartitionKey,
469 offset: i64,
470 max_bytes: usize,
471 ) -> Result<Vec<FetchedBatch>, ObjectLogError> {
472 let entries = self.sequencer.lookup(partition, offset)?;
473 let mut out = Vec::new();
474 let mut total = 0usize;
475 for e in entries {
476 if total >= max_bytes && !out.is_empty() {
477 break;
478 }
479 let batch = self.load_entry(&e).await?;
480 total += batch.payload.len();
481 out.push(batch);
482 }
483 Ok(out)
484 }
485
486 pub async fn fetch_stream<F>(
493 &self,
494 partition: &PartitionKey,
495 offset: i64,
496 mut visit: F,
497 ) -> Result<(), ObjectLogError>
498 where
499 F: FnMut(FetchedBatch) -> Result<(), ObjectLogError>,
500 {
501 let entries = self.sequencer.lookup(partition, offset)?;
502 for e in entries {
503 visit(self.load_entry(&e).await?)?;
504 }
505 Ok(())
506 }
507
508 async fn load_entry(&self, e: &IndexEntry) -> Result<FetchedBatch, ObjectLogError> {
509 let start = e.location.byte_start as u64;
510 let end = start + e.location.byte_len as u64;
511 let bytes = get_range_with_retries(&self.blob, &e.location.object_id, start..end)
512 .await?
513 .ok_or_else(|| ObjectLogError::MissingObject(e.location.object_id.clone()))?;
514 Ok(FetchedBatch {
515 base_offset: e.base_offset,
516 record_count: e.record_count,
517 payload: bytes,
518 })
519 }
520
521 pub async fn truncate_before(
524 &self,
525 partition: &PartitionKey,
526 offset: i64,
527 ) -> Result<(), ObjectLogError> {
528 let dead = self.sequencer.truncate_before(partition, offset)?;
529 for object_id in dead {
530 self.blob.delete(&object_id).await?;
531 }
532 Ok(())
533 }
534
535 pub async fn reap_orphans(
545 &self,
546 live: &HashSet<String>,
547 ) -> Result<Vec<String>, ObjectLogError> {
548 reap_orphans(self.blob.as_ref(), &self.key_prefix, live).await
549 }
550
551 pub fn buffer_stats(&self) -> BufferStats {
553 let q = self.shared.queue.lock().expect("poisoned");
554 BufferStats {
555 queued_bytes: q.bytes,
556 bytes_in_use: q.bytes_in_use,
557 queued_batches: q.items.len(),
558 max_buffered_bytes: self.shared.max_buffered_bytes,
559 }
560 }
561
562 pub fn pipeline_snapshot(&self) -> PipelineSnapshot {
564 let now = Instant::now();
565 let mut budget = self.shared.budget.lock().expect("poisoned");
566 budget.refill(now);
567 let effective = budget.effective_budget_per_sec;
568 let reason = if !budget.config.enabled {
569 EffectiveReason::Disabled
570 } else if budget.config.budget_per_sec_cap.is_some()
571 && budget
572 .config
573 .budget_per_sec_cap
574 .is_some_and(|c| (effective - c).abs() < 1e-9)
575 {
576 EffectiveReason::ConfigCap
577 } else if budget.ongoing_capacity.is_some() {
578 EffectiveReason::Ongoing
579 } else if budget.startup_capacity.is_some() {
580 EffectiveReason::StartupProbe
581 } else {
582 EffectiveReason::DefaultCapacity
583 };
584 let (queued, last_enq) = {
585 let q = self.shared.queue.lock().expect("poisoned");
586 (q.bytes, q.last_enqueue)
587 };
588 let early = budget.allow_early_flush(now, queued, last_enq);
589 let max_linger = self.shared.flush_config.linger;
590 let eff_linger = if !budget.config.enabled {
591 max_linger
592 } else if early || max_linger.is_zero() {
593 Duration::ZERO
594 } else {
595 max_linger
596 };
597 PipelineSnapshot {
598 budget_per_sec: EffectiveKnob {
599 configured: budget.config.budget_per_sec_cap,
600 startup_measured: budget.startup_capacity,
601 ongoing_measured: budget.ongoing_capacity,
602 effective,
603 reason,
604 },
605 effective_linger_ms: EffectiveKnob {
606 configured: Some(max_linger.as_millis() as u64),
607 startup_measured: None,
608 ongoing_measured: None,
609 effective: eff_linger.as_millis() as u64,
610 reason: if early {
611 EffectiveReason::Ongoing
612 } else {
613 EffectiveReason::Configured
614 },
615 },
616 max_linger_ms: max_linger.as_millis() as u64,
617 token_fill_ratio: budget.fill_ratio(),
618 media_ops_total: budget.media_ops_total,
619 overdraft_total: budget.overdraft_total,
620 flushes_total: budget.flushes_total,
621 undersized_deadline_flushes: budget.undersized_deadline_flushes,
622 predicted_media_ops: budget.predicted_media_ops,
623 budget_enabled: budget.config.enabled,
624 budget_mode: budget.config.mode,
625 }
626 }
627}
628
629pub async fn reap_orphans(
635 blob: &dyn BlobStore,
636 data_prefix: &str,
637 live: &HashSet<String>,
638) -> Result<Vec<String>, ObjectLogError> {
639 let keys = blob.list(data_prefix).await?;
640 let mut deleted = Vec::new();
641 for key in keys {
642 if live.contains(&key) {
643 continue;
644 }
645 blob.delete(&key).await?;
646 deleted.push(key);
647 }
648 Ok(deleted)
649}
650
651impl<S: Sequencer> Drop for LogEngine<S> {
652 fn drop(&mut self) {
653 {
654 let mut q = self.shared.queue.lock().expect("poisoned");
655 q.shutdown = true;
656 }
657 self.shared.cv.notify_all();
658 if let Some(t) = self.flush_thread.take() {
659 let _ = t.join();
660 }
661 }
662}
663
664async fn recover_data_object_counter(
667 blob: &dyn BlobStore,
668 prefix: &str,
669) -> Result<u64, ObjectLogError> {
670 let keys = blob.list(prefix).await?;
671 let mut max = 0u64;
672 for key in keys {
673 let Some(suffix) = key.strip_prefix(prefix) else {
674 continue;
675 };
676 if let Ok(n) = suffix.parse::<u64>() {
677 max = max.max(n);
678 }
679 }
680 Ok(max)
681}
682
683fn flush_loop<S>(
684 shared: Arc<Shared<S::Meta>>,
685 blob: Arc<dyn BlobStore>,
686 sequencer: Arc<S>,
687 config: FlushConfig,
688 prefix: String,
689) where
690 S: Sequencer + 'static,
691 S::Meta: Send + 'static,
692{
693 let max_inflight = config.max_inflight_flushes.max(1);
694 let commit_group_limit = if sequencer.supports_multi_object_commit() {
695 max_inflight
696 } else {
697 1
698 };
699 let worker_threads = std::env::var("OBJECT_LOG_FLUSH_RUNTIME_THREADS")
700 .ok()
701 .and_then(|v| v.parse::<usize>().ok())
702 .filter(|v| *v > 0)
703 .unwrap_or_else(|| max_inflight.min(8));
704 let rt = tokio::runtime::Builder::new_multi_thread()
705 .worker_threads(worker_threads)
706 .enable_all()
707 .build()
708 .expect("flush runtime");
709 let mut counter = match rt.block_on(recover_data_object_counter(blob.as_ref(), &prefix)) {
713 Ok(counter) => counter,
714 Err(error) => {
715 let _ = fail_engine_and_take_pending_uploads(&shared, &mut VecDeque::new(), 0, error);
718 return;
719 }
720 };
721 let mut pending: VecDeque<FlushWork<S::Meta>> = VecDeque::new();
722 let mut active_puts = 0usize;
723 let mut committing: Option<CommitJob> = None;
724 let mut shutdown = false;
725
726 loop {
727 while !shutdown && active_puts < max_inflight {
728 let wait_for_more = pending.is_empty() && committing.is_none();
731 match take_batch(&shared, config, wait_for_more) {
732 TakeBatch::Batch(batch) => {
733 counter += 1;
734 let concurrent = max_inflight > 1;
735 let work = start_flush_work(&rt, &blob, &prefix, counter, batch, concurrent);
736 if concurrent {
737 active_puts += 1;
738 }
739 pending.push_back(work);
740 }
741 TakeBatch::Empty => break,
742 TakeBatch::Shutdown => shutdown = true,
743 }
744 }
745
746 let mut made_progress = false;
747 for work in pending.iter_mut() {
748 let Some(put) = work.put.as_ref() else {
749 continue;
750 };
751 if !put.is_finished() {
752 continue;
753 }
754 let put = work.put.take().expect("put handle exists");
755 let elapsed = work.put_started.elapsed();
756 work.put_result = Some(match rt.block_on(put) {
757 Ok(Ok(())) => Ok(elapsed),
758 Ok(Err(e)) => Err(e),
759 Err(e) => Err(ObjectLogError::StorageUnavailable(format!(
760 "flush task failed: {e}"
761 ))),
762 });
763 active_puts = active_puts.saturating_sub(1);
764 made_progress = true;
765 }
766
767 if committing
768 .as_ref()
769 .is_some_and(|job| job.task.is_finished())
770 {
771 let job = committing.take().expect("completed commit exists");
772 match rt.block_on(job.task) {
773 Ok(released) => {
774 let mut q = shared.queue.lock().expect("poisoned");
775 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
776 shared.cv.notify_all();
777 }
778 Err(error) => {
779 let uploads = fail_engine_and_take_pending_uploads(
780 &shared,
781 &mut pending,
782 job.first_seq,
783 ObjectLogError::Sequencer(format!("commit worker failed: {error}")),
784 );
785 for upload in uploads {
789 let _ = rt.block_on(upload);
790 }
791 return;
792 }
793 }
794 made_progress = true;
795 }
796
797 if committing.is_none() {
798 if let Some(work) = take_ready_commit_group(&mut pending, commit_group_limit) {
799 if max_inflight == 1 {
800 let released = finish_flush_work(&shared, &blob, &sequencer, work);
802 let mut q = shared.queue.lock().expect("poisoned");
803 q.bytes_in_use = q.bytes_in_use.saturating_sub(released);
804 shared.cv.notify_all();
805 } else {
806 let first_seq = work
809 .batch
810 .iter()
811 .map(|p| p.seq)
812 .min()
813 .unwrap_or(work.max_seq);
814 let shared = shared.clone();
815 let blob = blob.clone();
816 let sequencer = sequencer.clone();
817 committing = Some(CommitJob {
818 task: rt.spawn_blocking(move || {
819 finish_flush_work(&shared, &blob, &sequencer, work)
820 }),
821 first_seq,
822 });
823 }
824 made_progress = true;
825 }
826 }
827
828 if pending.is_empty() && committing.is_none() {
829 {
830 let mut q = shared.queue.lock().expect("poisoned");
831 if q.items.is_empty() && q.bytes_in_use == 0 {
832 notify_flush_waiters(&mut q);
835 }
836 }
837 shared.cv.notify_all();
838 if shutdown {
839 let mut q = shared.queue.lock().expect("poisoned");
841 for (_, tx) in q.flush_waiters.drain(..) {
842 let _ = tx.send(Err(ObjectLogError::Sequencer(
843 "engine shutting down".into(),
844 )));
845 }
846 return;
847 }
848 continue;
849 }
850
851 if !made_progress {
852 std::thread::sleep(Duration::from_millis(1));
853 }
854 }
855}
856
857fn fail_engine_and_take_pending_uploads<M>(
860 shared: &Arc<Shared<M>>,
861 pending: &mut VecDeque<FlushWork<M>>,
862 first_seq: u64,
863 error: ObjectLogError,
864) -> Vec<TokioJoinHandle<Result<(), ObjectLogError>>> {
865 let mut uploads = Vec::new();
866 for mut work in pending.drain(..) {
867 if let Some(put) = work.put.take() {
868 uploads.push(put);
869 }
870 send_storage_error(&mut work.responders, error.clone());
871 }
872 let mut q = shared
874 .queue
875 .lock()
876 .unwrap_or_else(|poisoned| poisoned.into_inner());
877 q.shutdown = true;
878 q.first_failed.get_or_insert((first_seq, error.clone()));
879 for mut item in q.items.drain(..) {
880 if let Some(tx) = item.responder.take() {
881 let _ = tx.send(Err(error.clone()));
882 }
883 }
884 q.bytes = 0;
885 for (_, tx) in q.flush_waiters.drain(..) {
888 let _ = tx.send(Err(error.clone()));
889 }
890 q.force_flush = false;
891 shared.cv.notify_all();
892 uploads
893}
894
895fn take_ready_commit_group<M>(
898 pending: &mut VecDeque<FlushWork<M>>,
899 max_objects: usize,
900) -> Option<FlushWork<M>> {
901 pending.front()?.put_result.as_ref()?;
902 let mut group = pending.pop_front().expect("ready front exists");
903 if group.put_result.as_ref().is_some_and(Result::is_err) {
904 return Some(group);
905 }
906 while group.data_objects < max_objects.max(1) {
907 let Some(next) = pending.front() else { break };
908 if !next.put_result.as_ref().is_some_and(Result::is_ok)
909 || group.data_objects.saturating_add(next.data_objects) > max_objects
910 {
911 break;
912 }
913 let mut next = pending.pop_front().expect("ready successor exists");
914 group.batch.append(&mut next.batch);
915 group.locations.append(&mut next.locations);
916 group.responders.append(&mut next.responders);
917 group.bytes = group
918 .bytes
919 .checked_add(next.bytes)
920 .expect("bounded pending bytes");
921 group.max_seq = group.max_seq.max(next.max_seq);
922 group.data_objects += next.data_objects;
923 let next_elapsed = next.put_result.take().unwrap().unwrap();
924 let elapsed = group.put_result.as_mut().unwrap().as_mut().unwrap();
925 *elapsed = (*elapsed).max(next_elapsed);
926 }
927 Some(group)
928}
929
930fn effective_linger<M>(
934 shared: &Shared<M>,
935 config: FlushConfig,
936 queued_bytes: usize,
937 last_enqueue: Option<Instant>,
938 oldest_enqueue: Option<Instant>,
939) -> Duration {
940 if config.linger.is_zero() {
941 return Duration::ZERO;
942 }
943 let now = Instant::now();
944 if config.budget.enabled {
945 let mut budget = shared.budget.lock().expect("poisoned");
946 budget.refill(now);
947 if budget.allow_early_flush(now, queued_bytes, last_enqueue) {
948 budget.note_early_flush(now);
949 return Duration::ZERO;
950 }
951 }
952 if let Some(oldest) = oldest_enqueue {
954 let deadline = oldest + config.linger;
955 if now >= deadline {
956 return Duration::ZERO;
957 }
958 let until_deadline = deadline.saturating_duration_since(now);
959 if config.budget.enabled
961 && !config.budget.early_flush_idle.is_zero()
962 && let Some(last) = last_enqueue
963 {
964 let idle_at = last + config.budget.early_flush_idle;
965 if idle_at > now && idle_at < deadline {
966 return idle_at.saturating_duration_since(now);
967 }
968 }
969 return until_deadline;
970 }
971 config.linger
972}
973
974fn take_batch<M>(
975 shared: &Arc<Shared<M>>,
976 config: FlushConfig,
977 wait_when_empty: bool,
978) -> TakeBatch<M> {
979 let mut q = shared.queue.lock().expect("poisoned");
980 loop {
981 if q.items.is_empty() {
982 if q.shutdown {
983 return TakeBatch::Shutdown;
984 }
985 if !wait_when_empty {
986 return TakeBatch::Empty;
987 }
988 q = shared.cv.wait(q).expect("poisoned");
992 continue;
993 }
994
995 let linger = {
996 let queued = q.bytes;
997 let last_enq = q.last_enqueue;
998 let oldest = q.oldest_enqueue;
999 drop(q);
1000 let l = effective_linger(shared, config, queued, last_enq, oldest);
1001 q = shared.queue.lock().expect("poisoned");
1002 l
1003 };
1004
1005 let size_trigger = q.bytes >= config.max_bytes || q.items.len() >= config.max_batches;
1006 let force = q.force_flush;
1007 let triggered = q.shutdown || size_trigger || linger.is_zero() || force;
1008 if triggered {
1009 if !size_trigger && !q.shutdown && !q.items.is_empty() {
1010 let mut budget = shared.budget.lock().expect("poisoned");
1011 if q.bytes < config.max_bytes && q.items.len() < config.max_batches {
1012 budget.undersized_deadline_flushes += 1;
1013 }
1014 }
1015 break;
1016 }
1017
1018 let (guard, _timeout) = shared.cv.wait_timeout(q, linger).expect("poisoned");
1021 q = guard;
1022 }
1023
1024 let force_drain = q.force_flush;
1025 let mut items = Vec::new();
1026 let mut bytes = 0usize;
1027 while let Some(item) = q.items.pop_front() {
1030 q.bytes = q.bytes.saturating_sub(item.payload.len());
1031 bytes += item.payload.len();
1032 items.push(item);
1033 if bytes >= config.max_bytes {
1034 break;
1035 }
1036 if !force_drain && items.len() >= config.max_batches {
1037 break;
1038 }
1039 }
1040 q.oldest_enqueue = if q.items.is_empty() {
1041 None
1042 } else {
1043 Some(Instant::now())
1045 };
1046 if force_drain {
1048 q.force_flush = !q.items.is_empty() || !q.flush_waiters.is_empty();
1049 }
1050 if items.is_empty() {
1051 TakeBatch::Empty
1052 } else {
1053 TakeBatch::Batch(items)
1054 }
1055}
1056
1057fn notify_flush_waiters<M>(q: &mut Queue<M>) {
1058 let done = q.completed_through;
1059 let mut i = 0;
1060 while i < q.flush_waiters.len() {
1061 let barrier = q.flush_waiters[i].0;
1062 let failure = q
1063 .first_failed
1064 .as_ref()
1065 .filter(|(sequence, _)| *sequence <= barrier)
1066 .map(|(_, error)| error.clone());
1067 if barrier <= done || failure.is_some() {
1068 let (_, tx) = q.flush_waiters.swap_remove(i);
1069 let _ = tx.send(failure.map_or(Ok(()), Err));
1070 } else {
1071 i += 1;
1072 }
1073 }
1074 if q.flush_waiters.is_empty() {
1075 q.force_flush = false;
1076 } else if q.items.is_empty() && q.bytes_in_use == 0 {
1077 q.force_flush = true;
1079 } else {
1080 q.force_flush = true;
1081 }
1082}
1083
1084fn send_storage_error(responders: &mut [(Durability, Option<Responder>)], err: ObjectLogError) {
1085 for (_, tx) in responders.iter_mut() {
1086 if let Some(tx) = tx.take() {
1087 let _ = tx.send(Err(err.clone()));
1088 }
1089 }
1090}
1091
1092fn send_durable_acks(responders: &mut [(Durability, Option<Responder>)]) {
1093 for (durability, tx) in responders.iter_mut() {
1094 if *durability == Durability::Durable
1095 && let Some(tx) = tx.take()
1096 {
1097 let _ = tx.send(Ok(AppendOutcome {
1098 base_offset: None,
1099 last_offset: None,
1100 durable: true,
1101 sequenced: false,
1102 }));
1103 }
1104 }
1105}
1106
1107fn prepare_flush_work<M>(
1108 prefix: &str,
1109 counter: u64,
1110 mut batch: Vec<Pending<M>>,
1111) -> (FlushWork<M>, String, Vec<Bytes>) {
1112 let mut locations: Vec<BatchLocation> = Vec::with_capacity(batch.len());
1113 let mut chunks: Vec<Bytes> = Vec::with_capacity(batch.len());
1114 let key = format!("{prefix}{counter:020}");
1115 let mut offset = 0usize;
1116 for p in &batch {
1117 let start = offset as u32;
1118 offset += p.payload.len();
1119 chunks.push(p.payload.clone());
1120 locations.push(BatchLocation {
1121 object_id: key.clone(),
1122 byte_start: start,
1123 byte_len: p.payload.len() as u32,
1124 });
1125 }
1126 if std::env::var("OLOG_DEBUG_FLUSH_CONFIG").is_ok() {
1127 eprintln!(
1128 "object-log seal: key={key} batches={} bytes={offset}",
1129 batch.len()
1130 );
1131 }
1132 let responders: Vec<(Durability, Option<Responder>)> = batch
1133 .iter_mut()
1134 .map(|p| (p.durability, p.responder.take()))
1135 .collect();
1136 let max_seq = batch.iter().map(|p| p.seq).max().unwrap_or(0);
1137 let work = FlushWork {
1138 batch,
1139 locations,
1140 responders,
1141 bytes: offset,
1142 max_seq,
1143 data_objects: 1,
1144 put: None,
1145 put_started: Instant::now(),
1146 put_result: None,
1147 };
1148 (work, key, chunks)
1149}
1150
1151fn start_flush_work<M>(
1157 rt: &tokio::runtime::Runtime,
1158 blob: &Arc<dyn BlobStore>,
1159 prefix: &str,
1160 counter: u64,
1161 batch: Vec<Pending<M>>,
1162 concurrent: bool,
1163) -> FlushWork<M> {
1164 let (mut work, key, chunks) = prepare_flush_work(prefix, counter, batch);
1165 let blob = Arc::clone(blob);
1166 let _ = blob.take_media_op_stats();
1167 work.put_started = Instant::now();
1168 if concurrent {
1169 work.put =
1170 Some(rt.spawn(async move { put_chunks_with_retries(&blob, &key, chunks).await }));
1171 } else {
1172 let put_result = rt.block_on(put_chunks_with_retries(&blob, &key, chunks));
1173 let elapsed = work.put_started.elapsed();
1174 work.put_result = Some(match put_result {
1175 Ok(()) => Ok(elapsed),
1176 Err(e) => Err(e),
1177 });
1178 }
1179 work
1180}
1181
1182fn finish_flush_work<S, M>(
1183 shared: &Arc<Shared<M>>,
1184 blob: &Arc<dyn BlobStore>,
1185 sequencer: &Arc<S>,
1186 mut work: FlushWork<S::Meta>,
1187) -> usize
1188where
1189 S: Sequencer + 'static,
1190 S::Meta: Send + 'static,
1191{
1192 let release_bytes = work.bytes;
1193 let first_seq = work
1194 .batch
1195 .iter()
1196 .map(|pending| pending.seq)
1197 .min()
1198 .unwrap_or(work.max_seq);
1199 let timing = std::env::var("OLOG_DEBUG_FLUSH_TIMING").is_ok();
1203 let put_elapsed = match work.put_result.take().expect("put result is ready") {
1204 Ok(elapsed) => elapsed,
1205 Err(e) => {
1206 send_storage_error(&mut work.responders, e.clone());
1207 let mut q = shared.queue.lock().expect("poisoned");
1208 let done = work.max_seq;
1210 if done >= q.completed_through {
1211 q.completed_through = done;
1212 }
1213 q.first_failed.get_or_insert((first_seq, e));
1214 notify_flush_waiters(&mut q);
1215 shared.cv.notify_all();
1216 return release_bytes;
1217 }
1218 };
1219
1220 let mut media_ops = blob
1222 .take_media_op_stats()
1223 .map(|s| s.media_ops)
1224 .unwrap_or(work.data_objects as u64); send_durable_acks(&mut work.responders);
1228
1229 let commit_batches: Vec<CommitBatch<'_, S::Meta>> = work
1231 .batch
1232 .iter()
1233 .zip(work.locations.iter())
1234 .map(|(p, loc)| CommitBatch {
1235 partition: p.partition.clone(),
1236 record_count: p.record_count,
1237 location: loc.clone(),
1238 meta: &p.meta,
1239 })
1240 .collect();
1241
1242 let commit_started = timing.then(Instant::now);
1243 let _ = blob.take_media_op_stats();
1246 let mut commit_error = None;
1247 match sequencer.commit(&commit_batches) {
1248 Ok(outcomes) => {
1249 if let Some(commit_started) = commit_started {
1250 eprintln!(
1251 "object-log flush timing: bytes={} batches={} put_ms={} commit_ms={} objects={}",
1252 work.bytes,
1253 commit_batches.len(),
1254 put_elapsed.as_millis(),
1255 commit_started.elapsed().as_millis(),
1256 work.data_objects
1257 );
1258 }
1259 if let Some(s) = blob.take_media_op_stats() {
1260 media_ops = media_ops.saturating_add(s.media_ops);
1261 }
1262 for (outcome, (_, tx)) in outcomes.into_iter().zip(work.responders.iter_mut()) {
1263 if let Some(tx) = tx.take() {
1264 let (base, last) = match outcome {
1265 CommitOutcome::Assigned {
1266 base_offset,
1267 record_count,
1268 } => (
1269 Some(base_offset),
1270 Some(base_offset + record_count as i64 - 1),
1271 ),
1272 CommitOutcome::Duplicate { base_offset } => (Some(base_offset), None),
1273 };
1274 let _ = tx.send(Ok(AppendOutcome {
1275 base_offset: base,
1276 last_offset: last,
1277 durable: true,
1278 sequenced: true,
1279 }));
1280 }
1281 }
1282 }
1283 Err(e) => {
1284 if let Some(s) = blob.take_media_op_stats() {
1285 media_ops = media_ops.saturating_add(s.media_ops);
1286 }
1287 send_storage_error(&mut work.responders, e.clone());
1288 commit_error = Some(e);
1289 }
1290 }
1291
1292 {
1293 let mut budget = shared.budget.lock().expect("poisoned");
1294 budget.consume_after_flush(media_ops, Instant::now());
1295 }
1296 {
1297 let mut q = shared.queue.lock().expect("poisoned");
1298 if let Some(error) = commit_error {
1299 q.first_failed.get_or_insert((first_seq, error));
1300 }
1301 if work.max_seq >= q.completed_through {
1302 q.completed_through = work.max_seq;
1303 }
1304 notify_flush_waiters(&mut q);
1305 }
1306 shared.cv.notify_all();
1307
1308 release_bytes
1309}
1310
1311#[cfg(test)]
1312mod flush_barrier_tests {
1313 use super::*;
1314
1315 #[test]
1316 fn failure_only_rejects_barriers_covering_its_enqueue_position() {
1317 let error = ObjectLogError::Sequencer("manifest failed".into());
1318 let mut queue = Queue::<()> {
1319 items: VecDeque::new(),
1320 bytes: 0,
1321 bytes_in_use: 1,
1322 shutdown: false,
1323 next_seq: 4,
1324 completed_through: 2,
1325 first_failed: Some((2, error.clone())),
1326 force_flush: true,
1327 flush_waiters: Vec::new(),
1328 last_enqueue: None,
1329 oldest_enqueue: None,
1330 };
1331 let mut receivers = Vec::new();
1332 for barrier in [1, 2, 3] {
1333 let (tx, rx) = oneshot::channel();
1334 queue.flush_waiters.push((barrier, tx));
1335 receivers.push(rx);
1336 }
1337 notify_flush_waiters(&mut queue);
1338 assert_eq!(receivers[0].try_recv().unwrap(), Ok(()));
1339 assert_eq!(receivers[1].try_recv().unwrap(), Err(error.clone()));
1340 assert_eq!(receivers[2].try_recv().unwrap(), Err(error));
1341 assert!(queue.flush_waiters.is_empty());
1342 }
1343}
1344
1345#[cfg(test)]
1346mod ready_commit_group_tests {
1347 use super::*;
1348 use crate::{ManifestSequencer, MemoryBlobStore};
1349 use async_trait::async_trait;
1350 use std::ops::Range;
1351
1352 fn work(
1353 n: u64,
1354 durability: Durability,
1355 ) -> (
1356 FlushWork<()>,
1357 String,
1358 Vec<Bytes>,
1359 oneshot::Receiver<Result<AppendOutcome, ObjectLogError>>,
1360 ) {
1361 let (tx, rx) = oneshot::channel();
1362 let (mut work, key, chunks) = prepare_flush_work(
1363 "data/",
1364 n,
1365 vec![Pending {
1366 partition: PartitionKey("shared".into()),
1367 record_count: 1,
1368 payload: Bytes::from(format!("body-{n}")),
1369 meta: (),
1370 durability,
1371 responder: Some(tx),
1372 seq: n,
1373 }],
1374 );
1375 work.put_result = Some(Ok(Duration::from_millis(n)));
1376 (work, key, chunks, rx)
1377 }
1378
1379 #[test]
1380 fn ready_groups_are_bounded_and_do_not_cross_unfinished_or_failed_puts() {
1381 let mut pending = (1..=5)
1382 .map(|n| work(n, Durability::Sequenced).0)
1383 .collect::<VecDeque<_>>();
1384 let original_bytes: usize = pending.iter().map(|w| w.bytes).sum();
1385 let mut released = 0;
1386 for expected in [vec![1, 2], vec![3, 4], vec![5]] {
1387 let group = take_ready_commit_group(&mut pending, 2).unwrap();
1388 assert_eq!(
1389 group.batch.iter().map(|p| p.seq).collect::<Vec<_>>(),
1390 expected
1391 );
1392 assert_eq!(group.data_objects, expected.len());
1393 assert_eq!(group.responders.len(), expected.len());
1394 assert_eq!(group.max_seq, *expected.last().unwrap());
1395 for (location, n) in group.locations.iter().zip(&expected) {
1396 assert_eq!(location.object_id, format!("data/{n:020}"));
1397 assert_eq!(location.byte_start, 0);
1398 assert_eq!(location.byte_len as usize, format!("body-{n}").len());
1399 }
1400 released += group.bytes;
1401 }
1402 assert!(pending.is_empty());
1403 assert_eq!(released, original_bytes);
1404
1405 let mut first = work(1, Durability::Sequenced).0;
1406 first.put_result = None;
1407 let mut pending = VecDeque::from([first, work(2, Durability::Sequenced).0]);
1408 assert!(take_ready_commit_group(&mut pending, 8).is_none());
1409 assert_eq!(
1410 pending.len(),
1411 2,
1412 "a later ready PUT cannot pass an unfinished head"
1413 );
1414 pending[0].put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1415 let failed = take_ready_commit_group(&mut pending, 8).unwrap();
1416 assert_eq!(failed.data_objects, 1);
1417 assert!(failed.put_result.unwrap().is_err());
1418 assert_eq!(pending[0].max_seq, 2);
1419
1420 let mut failed = work(2, Durability::Sequenced).0;
1421 failed.put_result = Some(Err(ObjectLogError::StorageUnavailable("failed PUT".into())));
1422 let mut pending = VecDeque::from([
1423 work(1, Durability::Sequenced).0,
1424 failed,
1425 work(3, Durability::Sequenced).0,
1426 ]);
1427 assert_eq!(take_ready_commit_group(&mut pending, 8).unwrap().max_seq, 1);
1428 assert_eq!(
1429 pending.len(),
1430 2,
1431 "a success group cannot cross a failed PUT"
1432 );
1433 let mut pending = VecDeque::from([
1434 work(1, Durability::Sequenced).0,
1435 work(2, Durability::Sequenced).0,
1436 ]);
1437 assert_eq!(
1438 take_ready_commit_group(&mut pending, 1)
1439 .unwrap()
1440 .data_objects,
1441 1
1442 );
1443 assert_eq!(
1444 pending.len(),
1445 1,
1446 "one-object sequencers retain their call boundary"
1447 );
1448 }
1449
1450 struct GatedManifestStore {
1451 inner: MemoryBlobStore,
1452 entered: std::sync::mpsc::SyncSender<()>,
1453 release: Mutex<Option<oneshot::Receiver<()>>>,
1454 fail: bool,
1455 panic_on_commit: bool,
1456 }
1457
1458 #[async_trait]
1459 impl BlobStore for GatedManifestStore {
1460 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1461 if key.starts_with("manifest/") {
1462 let release = self.release.lock().unwrap().take();
1463 if let Some(release) = release {
1464 self.entered.send(()).unwrap();
1465 release.await.unwrap();
1466 assert!(!self.panic_on_commit, "injected commit panic");
1467 if self.fail {
1468 return Err(ObjectLogError::StorageUnavailable(
1469 "manifest failure".into(),
1470 ));
1471 }
1472 }
1473 }
1474 self.inner.put(key, value).await
1475 }
1476 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1477 self.inner.get(key).await
1478 }
1479 async fn get_range(
1480 &self,
1481 key: &str,
1482 range: Range<u64>,
1483 ) -> Result<Option<Bytes>, ObjectLogError> {
1484 self.inner.get_range(key, range).await
1485 }
1486 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1487 self.inner.list(prefix).await
1488 }
1489 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1490 self.inner.delete(key).await
1491 }
1492 fn take_media_op_stats(&self) -> Option<crate::MediaOpStats> {
1493 self.inner.take_media_op_stats()
1494 }
1495 }
1496
1497 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1498 async fn grouped_manifest_controls_acknowledgements_and_reopens_exact_locations() {
1499 for fail in [false, true] {
1500 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1501 let (release_tx, release_rx) = oneshot::channel();
1502 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1503 inner: MemoryBlobStore::new(),
1504 entered: entered_tx,
1505 release: Mutex::new(Some(release_rx)),
1506 fail,
1507 panic_on_commit: false,
1508 });
1509 let sequencer = Arc::new(
1510 ManifestSequencer::open(blob.clone(), "manifest/")
1511 .await
1512 .unwrap(),
1513 );
1514 assert!(sequencer.supports_multi_object_commit());
1515 let (first, first_key, first_chunks, mut sequenced_rx) = work(1, Durability::Sequenced);
1516 let (mut second, second_key, second_chunks, mut durable_rx) =
1517 work(2, Durability::Durable);
1518 second.batch[0].partition = PartitionKey("other".into());
1519 let (third, third_key, third_chunks, mut third_rx) = work(3, Durability::Sequenced);
1520 blob.put_chunks(&first_key, first_chunks).await.unwrap();
1521 blob.put_chunks(&second_key, second_chunks).await.unwrap();
1522 blob.put_chunks(&third_key, third_chunks).await.unwrap();
1523 let mut pending = VecDeque::from([first, second, third]);
1524 let group = take_ready_commit_group(&mut pending, 8).unwrap();
1525 let bytes = group.bytes;
1526 assert_eq!(group.data_objects, 3);
1527 let (flush_tx, mut flush_rx) = oneshot::channel();
1528 let mut config = FlushConfig::default();
1529 config.budget.enabled = false;
1530 let shared: Arc<Shared<()>> = Arc::new(Shared {
1531 queue: Mutex::new(Queue {
1532 items: VecDeque::new(),
1533 bytes: 0,
1534 bytes_in_use: bytes,
1535 shutdown: false,
1536 next_seq: 4,
1537 completed_through: 0,
1538 first_failed: None,
1539 force_flush: true,
1540 flush_waiters: vec![(3, flush_tx)],
1541 last_enqueue: None,
1542 oldest_enqueue: None,
1543 }),
1544 cv: Condvar::new(),
1545 max_buffered_bytes: bytes,
1546 budget: Mutex::new(BudgetRuntime::new(config.budget)),
1547 flush_config: config,
1548 });
1549 let worker = {
1550 let shared = shared.clone();
1551 let blob = blob.clone();
1552 let sequencer = sequencer.clone();
1553 std::thread::spawn(move || finish_flush_work(&shared, &blob, &sequencer, group))
1554 };
1555 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1556 assert!(matches!(
1557 sequenced_rx.try_recv(),
1558 Err(oneshot::error::TryRecvError::Empty)
1559 ));
1560 assert!(matches!(
1561 third_rx.try_recv(),
1562 Err(oneshot::error::TryRecvError::Empty)
1563 ));
1564 assert!(matches!(
1565 flush_rx.try_recv(),
1566 Err(oneshot::error::TryRecvError::Empty)
1567 ));
1568 let durable = durable_rx.try_recv().unwrap().unwrap();
1569 assert!(durable.durable && !durable.sequenced);
1570 release_tx.send(()).unwrap();
1571 assert_eq!(worker.join().unwrap(), bytes);
1572 if fail {
1573 assert!(sequenced_rx.await.unwrap().is_err());
1574 assert!(third_rx.await.unwrap().is_err());
1575 assert!(flush_rx.await.unwrap().is_err());
1576 assert_eq!(
1577 sequencer
1578 .high_watermark(&PartitionKey("shared".into()))
1579 .unwrap(),
1580 0
1581 );
1582 assert!(blob.list("manifest/").await.unwrap().is_empty());
1583 } else {
1584 assert_eq!(sequenced_rx.await.unwrap().unwrap().base_offset, Some(0));
1585 assert_eq!(third_rx.await.unwrap().unwrap().base_offset, Some(1));
1586 flush_rx.await.unwrap().unwrap();
1587 assert_eq!(blob.list("manifest/").await.unwrap().len(), 1);
1588 }
1589 assert_eq!(blob.list("data/").await.unwrap().len(), 3);
1590 let reopened = ManifestSequencer::open(blob.clone(), "manifest/")
1591 .await
1592 .unwrap();
1593 for (partition, objects) in [("shared", vec![1, 3]), ("other", vec![2])] {
1594 let entries = reopened.lookup(&PartitionKey(partition.into()), 0).unwrap();
1595 assert_eq!(entries.len(), if fail { 0 } else { objects.len() });
1596 for (offset, (entry, object)) in entries.iter().zip(objects).enumerate() {
1597 assert_eq!(entry.base_offset, offset as i64);
1598 let loc = &entry.location;
1599 assert_eq!(loc.object_id, format!("data/{object:020}"));
1600 let body = blob
1601 .get_range(
1602 &loc.object_id,
1603 loc.byte_start as u64..(loc.byte_start + loc.byte_len) as u64,
1604 )
1605 .await
1606 .unwrap()
1607 .unwrap();
1608 assert_eq!(body, format!("body-{object}"));
1609 }
1610 }
1611 }
1612 }
1613
1614 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1615 async fn uploads_continue_while_the_ordered_manifest_commit_is_blocked() {
1616 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1617 let (release_tx, release_rx) = oneshot::channel();
1618 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
1619 inner: MemoryBlobStore::new(),
1620 entered: entered_tx,
1621 release: Mutex::new(Some(release_rx)),
1622 fail: false,
1623 panic_on_commit: false,
1624 });
1625 let sequencer = Arc::new(
1626 ManifestSequencer::open(blob.clone(), "manifest/")
1627 .await
1628 .unwrap(),
1629 );
1630 let mut config = FlushConfig::default();
1631 config.max_batches = 1;
1632 config.max_inflight_flushes = 4;
1633 config.linger = Duration::ZERO;
1634 config.budget.enabled = false;
1635 let engine = Arc::new(LogEngine::new(blob.clone(), sequencer, config, "data/"));
1636 let first = {
1637 let engine = engine.clone();
1638 tokio::spawn(async move {
1639 engine
1640 .produce(
1641 PartitionKey("p".into()),
1642 Bytes::from_static(b"first"),
1643 1,
1644 (),
1645 Durability::Sequenced,
1646 )
1647 .await
1648 })
1649 };
1650 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1651 for n in 1..4 {
1652 engine
1653 .produce(
1654 PartitionKey("p".into()),
1655 Bytes::from(format!("later-{n}")),
1656 1,
1657 (),
1658 Durability::Buffered,
1659 )
1660 .await
1661 .unwrap();
1662 }
1663 let upload_progress = tokio::time::timeout(Duration::from_secs(1), async {
1664 loop {
1665 if blob.list("data/").await.unwrap().len() == 4 {
1666 break;
1667 }
1668 tokio::time::sleep(Duration::from_millis(1)).await;
1669 }
1670 })
1671 .await
1672 .is_ok();
1673 let early_ack = first.is_finished();
1674 release_tx.send(()).unwrap();
1676 first.await.unwrap().unwrap();
1677 engine.flush().await.unwrap();
1678 let rows = engine
1679 .fetch(&PartitionKey("p".into()), 0, 1024)
1680 .await
1681 .unwrap();
1682 assert_eq!(
1683 rows.iter().map(|r| r.base_offset).collect::<Vec<_>>(),
1684 vec![0, 1, 2, 3]
1685 );
1686 drop(engine);
1687 assert!(
1688 !early_ack,
1689 "sequenced acknowledgement preceded manifest durability"
1690 );
1691 assert!(
1692 upload_progress,
1693 "blocked manifest prevented independent data uploads"
1694 );
1695 }
1696
1697 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1698 async fn commit_completion_releases_admission_bytes_without_more_uploads() {
1699 let blob: Arc<dyn BlobStore> = Arc::new(MemoryBlobStore::new());
1700 let sequencer = Arc::new(
1701 ManifestSequencer::open(blob.clone(), "manifest/")
1702 .await
1703 .unwrap(),
1704 );
1705 let mut config = FlushConfig::default();
1706 config.max_bytes = 4;
1707 config.max_buffered_bytes = 4;
1708 config.max_batches = 1;
1709 config.max_inflight_flushes = 4;
1710 config.linger = Duration::ZERO;
1711 config.budget.enabled = false;
1712 let engine = LogEngine::new(blob, sequencer, config, "data/");
1713 engine
1714 .produce(
1715 PartitionKey("p".into()),
1716 Bytes::from_static(b"full"),
1717 1,
1718 (),
1719 Durability::Sequenced,
1720 )
1721 .await
1722 .unwrap();
1723 tokio::time::timeout(Duration::from_secs(5), async {
1724 while engine.buffer_stats().bytes_in_use != 0 {
1725 tokio::time::sleep(Duration::from_millis(1)).await;
1726 }
1727 })
1728 .await
1729 .expect("completed committer stranded admission bytes");
1730 engine
1731 .produce(
1732 PartitionKey("p".into()),
1733 Bytes::from_static(b"next"),
1734 1,
1735 (),
1736 Durability::Sequenced,
1737 )
1738 .await
1739 .unwrap();
1740 engine.flush().await.unwrap();
1741 assert_eq!(
1742 engine
1743 .fetch(&PartitionKey("p".into()), 0, 100)
1744 .await
1745 .unwrap()
1746 .len(),
1747 2
1748 );
1749 }
1750
1751 struct FailureInjectingStore {
1752 inner: Arc<dyn BlobStore>,
1753 second_upload_gate: Mutex<Option<(std::sync::mpsc::SyncSender<()>, oneshot::Receiver<()>)>>,
1754 second_upload_completed: std::sync::atomic::AtomicBool,
1755 fail_data_listing: std::sync::atomic::AtomicBool,
1756 }
1757
1758 impl FailureInjectingStore {
1759 fn new(inner: Arc<dyn BlobStore>) -> Self {
1760 Self {
1761 inner,
1762 second_upload_gate: Mutex::new(None),
1763 second_upload_completed: std::sync::atomic::AtomicBool::new(false),
1764 fail_data_listing: std::sync::atomic::AtomicBool::new(false),
1765 }
1766 }
1767 }
1768
1769 #[async_trait]
1770 impl BlobStore for FailureInjectingStore {
1771 async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
1772 let gate = if key == "data/00000000000000000002" {
1773 self.second_upload_gate.lock().unwrap().take()
1774 } else {
1775 None
1776 };
1777 if let Some((entered, release)) = gate {
1778 entered.send(()).unwrap();
1779 release.await.unwrap();
1780 let result = self.inner.put(key, value).await;
1781 if result.is_ok() {
1782 self.second_upload_completed
1783 .store(true, std::sync::atomic::Ordering::SeqCst);
1784 }
1785 return result;
1786 }
1787 self.inner.put(key, value).await
1788 }
1789 async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
1790 self.inner.get(key).await
1791 }
1792 async fn get_range(
1793 &self,
1794 key: &str,
1795 range: Range<u64>,
1796 ) -> Result<Option<Bytes>, ObjectLogError> {
1797 self.inner.get_range(key, range).await
1798 }
1799 async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
1800 if prefix == "data/"
1801 && self
1802 .fail_data_listing
1803 .load(std::sync::atomic::Ordering::SeqCst)
1804 {
1805 return Err(ObjectLogError::StorageUnavailable(
1806 "injected object listing failure".into(),
1807 ));
1808 }
1809 self.inner.list(prefix).await
1810 }
1811 async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
1812 self.inner.delete(key).await
1813 }
1814 }
1815
1816 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1817 async fn failed_counter_listing_cannot_overwrite_existing_log_objects() {
1818 let inner = Arc::new(MemoryBlobStore::new());
1819 let key = "data/00000000000000000001";
1820 let original = Bytes::from_static(b"existing immutable object");
1821 inner.put(key, original.clone()).await.unwrap();
1822 let store = Arc::new(FailureInjectingStore::new(inner.clone()));
1823 store
1824 .fail_data_listing
1825 .store(true, std::sync::atomic::Ordering::SeqCst);
1826 let mut config = FlushConfig::default();
1827 config.max_batches = 1;
1828 config.max_inflight_flushes = 4;
1829 config.linger = Duration::ZERO;
1830 config.budget.enabled = false;
1831 let engine = LogEngine::new(
1832 store,
1833 Arc::new(crate::InMemorySequencer::new()),
1834 config,
1835 "data/",
1836 );
1837 let result = tokio::time::timeout(
1838 Duration::from_secs(5),
1839 engine.produce(
1840 PartitionKey("p".into()),
1841 Bytes::from_static(b"must not overwrite"),
1842 1,
1843 (),
1844 Durability::Sequenced,
1845 ),
1846 )
1847 .await;
1848 let flushed = tokio::time::timeout(Duration::from_secs(5), engine.flush()).await;
1849 tokio::task::spawn_blocking(move || drop(engine))
1850 .await
1851 .unwrap();
1852 let retained = inner.get(key).await.unwrap();
1853 assert_eq!(
1854 retained,
1855 Some(original),
1856 "failed listing must preserve sealed bytes"
1857 );
1858 assert!(
1859 result.unwrap().is_err(),
1860 "failed prefix recovery must reject admission"
1861 );
1862 assert!(
1863 flushed.unwrap().is_err(),
1864 "flush must report failed initialization"
1865 );
1866 assert_eq!(inner.list("data/").await.unwrap(), vec![key.to_string()]);
1867 }
1868
1869 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1870 async fn engines_commit_and_close_independently() {
1871 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
1872 let (release_tx, release_rx) = oneshot::channel();
1873 let a_blob = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1874 inner: MemoryBlobStore::new(),
1875 entered: entered_tx,
1876 release: Mutex::new(Some(release_rx)),
1877 fail: false,
1878 panic_on_commit: false,
1879 })));
1880 let b_blob = Arc::new(FailureInjectingStore::new(Arc::new(MemoryBlobStore::new())));
1881 let mut config = FlushConfig::default();
1882 config.max_batches = 1;
1883 config.max_inflight_flushes = 4;
1884 config.linger = Duration::ZERO;
1885 config.budget.enabled = false;
1886 let a_seq = Arc::new(
1887 ManifestSequencer::open(a_blob.clone(), "manifest/")
1888 .await
1889 .unwrap(),
1890 );
1891 let b_seq = Arc::new(
1892 ManifestSequencer::open(b_blob.clone(), "manifest/")
1893 .await
1894 .unwrap(),
1895 );
1896 let a = LogEngine::new(a_blob.clone(), a_seq, config, "data/");
1897 a.produce(
1898 PartitionKey("p".into()),
1899 Bytes::from_static(b"a-first"),
1900 1,
1901 (),
1902 Durability::Buffered,
1903 )
1904 .await
1905 .unwrap();
1906 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
1907 let b = LogEngine::new(b_blob.clone(), b_seq, config, "data/");
1908 let b_result = tokio::time::timeout(
1909 Duration::from_secs(5),
1910 b.produce(
1911 PartitionKey("p".into()),
1912 Bytes::from_static(b"b-first"),
1913 1,
1914 (),
1915 Durability::Sequenced,
1916 ),
1917 )
1918 .await;
1919 let mut closing_b = tokio::task::spawn_blocking(move || drop(b));
1920 let b_closed = tokio::time::timeout(Duration::from_secs(5), &mut closing_b).await;
1921 release_tx.send(()).unwrap();
1923 if b_closed.is_err() {
1924 closing_b.await.unwrap();
1925 }
1926 assert!(
1927 b_closed.is_ok(),
1928 "closing B waited for A's blocked manifest"
1929 );
1930 assert_eq!(b_result.unwrap().unwrap().base_offset, Some(0));
1931 a.flush().await.unwrap();
1932 assert_eq!(
1933 a.produce(
1934 PartitionKey("p".into()),
1935 Bytes::from_static(b"a-second"),
1936 1,
1937 (),
1938 Durability::Sequenced
1939 )
1940 .await
1941 .unwrap()
1942 .base_offset,
1943 Some(1)
1944 );
1945 let rows = a.fetch(&PartitionKey("p".into()), 0, 100).await.unwrap();
1946 assert_eq!(
1947 rows.iter().map(|r| r.payload.clone()).collect::<Vec<_>>(),
1948 vec![
1949 Bytes::from_static(b"a-first"),
1950 Bytes::from_static(b"a-second")
1951 ]
1952 );
1953 tokio::task::spawn_blocking(move || drop(a)).await.unwrap();
1954 }
1955
1956 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1957 async fn failed_engine_drains_accepted_uploads_before_close_and_reopen() {
1958 let (manifest_entered_tx, manifest_entered_rx) = std::sync::mpsc::sync_channel(1);
1959 let (manifest_release_tx, manifest_release_rx) = oneshot::channel();
1960 let (upload_entered_tx, upload_entered_rx) = std::sync::mpsc::sync_channel(1);
1961 let (upload_release_tx, upload_release_rx) = oneshot::channel();
1962 let store = Arc::new(FailureInjectingStore::new(Arc::new(GatedManifestStore {
1963 inner: MemoryBlobStore::new(),
1964 entered: manifest_entered_tx,
1965 release: Mutex::new(Some(manifest_release_rx)),
1966 fail: false,
1967 panic_on_commit: true,
1968 })));
1969 *store.second_upload_gate.lock().unwrap() = Some((upload_entered_tx, upload_release_rx));
1970 let mut config = FlushConfig::default();
1971 config.max_batches = 1;
1972 config.max_inflight_flushes = 4;
1973 config.linger = Duration::ZERO;
1974 config.budget.enabled = false;
1975 let sequencer = Arc::new(
1976 ManifestSequencer::open(store.clone(), "manifest/")
1977 .await
1978 .unwrap(),
1979 );
1980 let engine = LogEngine::new(store.clone(), sequencer, config, "data/");
1981 engine
1982 .produce(
1983 PartitionKey("p".into()),
1984 Bytes::from_static(b"first"),
1985 1,
1986 (),
1987 Durability::Buffered,
1988 )
1989 .await
1990 .unwrap();
1991 manifest_entered_rx
1992 .recv_timeout(Duration::from_secs(5))
1993 .unwrap();
1994 engine
1995 .produce(
1996 PartitionKey("p".into()),
1997 Bytes::from_static(b"second"),
1998 1,
1999 (),
2000 Durability::Buffered,
2001 )
2002 .await
2003 .unwrap();
2004 upload_entered_rx
2005 .recv_timeout(Duration::from_secs(5))
2006 .unwrap();
2007 manifest_release_tx.send(()).unwrap();
2008 tokio::time::timeout(Duration::from_secs(5), async {
2009 while !engine.shared.queue.lock().unwrap().shutdown {
2010 tokio::time::sleep(Duration::from_millis(1)).await;
2011 }
2012 })
2013 .await
2014 .unwrap();
2015 let flush_failed = engine.flush().await.is_err();
2016 let upload_was_owned = upload_release_tx.send(()).is_ok();
2019 tokio::time::timeout(
2020 Duration::from_secs(5),
2021 tokio::task::spawn_blocking(move || drop(engine)),
2022 )
2023 .await
2024 .unwrap()
2025 .unwrap();
2026 assert!(flush_failed);
2027 assert!(upload_was_owned);
2028 assert!(
2029 store
2030 .second_upload_completed
2031 .load(std::sync::atomic::Ordering::SeqCst)
2032 );
2033 assert_eq!(store.list("data/").await.unwrap().len(), 2);
2034 let sequencer = Arc::new(
2035 ManifestSequencer::open(store.clone(), "manifest/")
2036 .await
2037 .unwrap(),
2038 );
2039 let reopened = LogEngine::new(store.clone(), sequencer, config, "data/");
2040 assert!(
2041 reopened
2042 .fetch(&PartitionKey("p".into()), 0, 100)
2043 .await
2044 .unwrap()
2045 .is_empty()
2046 );
2047 assert_eq!(
2048 reopened
2049 .produce(
2050 PartitionKey("p".into()),
2051 Bytes::from_static(b"after-reopen"),
2052 1,
2053 (),
2054 Durability::Sequenced
2055 )
2056 .await
2057 .unwrap()
2058 .base_offset,
2059 Some(0)
2060 );
2061 let rows = reopened
2062 .fetch(&PartitionKey("p".into()), 0, 100)
2063 .await
2064 .unwrap();
2065 assert_eq!(rows.len(), 1);
2066 assert_eq!(rows[0].payload, Bytes::from_static(b"after-reopen"));
2067 assert_eq!(store.list("data/").await.unwrap().len(), 3);
2068 tokio::task::spawn_blocking(move || drop(reopened))
2069 .await
2070 .unwrap();
2071 }
2072
2073 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2074 async fn shutdown_drains_a_blocked_committer_and_queued_buffered_work() {
2075 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2076 let (release_tx, release_rx) = oneshot::channel();
2077 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2078 inner: MemoryBlobStore::new(),
2079 entered: entered_tx,
2080 release: Mutex::new(Some(release_rx)),
2081 fail: false,
2082 panic_on_commit: false,
2083 });
2084 let sequencer = Arc::new(
2085 ManifestSequencer::open(blob.clone(), "manifest/")
2086 .await
2087 .unwrap(),
2088 );
2089 let mut config = FlushConfig::default();
2090 config.max_batches = 1;
2091 config.max_inflight_flushes = 4;
2092 config.linger = Duration::ZERO;
2093 config.budget.enabled = false;
2094 let engine = LogEngine::new(blob.clone(), sequencer, config, "data/");
2095 engine
2096 .produce(
2097 PartitionKey("p".into()),
2098 Bytes::from_static(b"first"),
2099 1,
2100 (),
2101 Durability::Buffered,
2102 )
2103 .await
2104 .unwrap();
2105 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2106 engine
2107 .produce(
2108 PartitionKey("p".into()),
2109 Bytes::from_static(b"second"),
2110 1,
2111 (),
2112 Durability::Buffered,
2113 )
2114 .await
2115 .unwrap();
2116 let shared = engine.shared.clone();
2117 let dropping = tokio::task::spawn_blocking(move || drop(engine));
2118 tokio::time::timeout(Duration::from_secs(5), async {
2119 while !shared.queue.lock().unwrap().shutdown {
2120 tokio::time::sleep(Duration::from_millis(1)).await;
2121 }
2122 })
2123 .await
2124 .unwrap();
2125 let premature = dropping.is_finished();
2126 release_tx.send(()).unwrap();
2127 tokio::time::timeout(Duration::from_secs(5), dropping)
2128 .await
2129 .unwrap()
2130 .unwrap();
2131 assert!(!premature, "shutdown returned before manifest durability");
2132 let reopened = Arc::new(
2133 ManifestSequencer::open(blob.clone(), "manifest/")
2134 .await
2135 .unwrap(),
2136 );
2137 let engine = LogEngine::new(blob, reopened, config, "data/");
2138 let rows = engine
2139 .fetch(&PartitionKey("p".into()), 0, 100)
2140 .await
2141 .unwrap();
2142 assert_eq!(
2143 rows.iter().map(|r| r.payload.as_ref()).collect::<Vec<_>>(),
2144 vec![b"first".as_slice(), b"second".as_slice()]
2145 );
2146 }
2147
2148 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2149 async fn panicked_committer_closes_admission_and_fails_flush() {
2150 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2151 let (release_tx, release_rx) = oneshot::channel();
2152 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2153 inner: MemoryBlobStore::new(),
2154 entered: entered_tx,
2155 release: Mutex::new(Some(release_rx)),
2156 fail: false,
2157 panic_on_commit: true,
2158 });
2159 let sequencer = Arc::new(
2160 ManifestSequencer::open(blob.clone(), "manifest/")
2161 .await
2162 .unwrap(),
2163 );
2164 let mut config = FlushConfig::default();
2165 config.max_batches = 1;
2166 config.max_inflight_flushes = 4;
2167 config.linger = Duration::ZERO;
2168 config.budget.enabled = false;
2169 let engine = Arc::new(LogEngine::new(blob, sequencer, config, "data/"));
2170 let first = {
2171 let engine = engine.clone();
2172 tokio::spawn(async move {
2173 engine
2174 .produce(
2175 PartitionKey("p".into()),
2176 Bytes::from_static(b"first"),
2177 1,
2178 (),
2179 Durability::Sequenced,
2180 )
2181 .await
2182 })
2183 };
2184 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2185 engine
2186 .produce(
2187 PartitionKey("p".into()),
2188 Bytes::from_static(b"later"),
2189 1,
2190 (),
2191 Durability::Buffered,
2192 )
2193 .await
2194 .unwrap();
2195 let flushing = {
2196 let engine = engine.clone();
2197 tokio::spawn(async move { engine.flush().await })
2198 };
2199 release_tx.send(()).unwrap();
2200 assert!(
2201 tokio::time::timeout(Duration::from_secs(5), first)
2202 .await
2203 .unwrap()
2204 .unwrap()
2205 .is_err()
2206 );
2207 assert!(
2208 tokio::time::timeout(Duration::from_secs(5), flushing)
2209 .await
2210 .unwrap()
2211 .unwrap()
2212 .is_err()
2213 );
2214 tokio::time::timeout(Duration::from_secs(5), async {
2215 while !engine.shared.queue.lock().unwrap().shutdown {
2216 tokio::time::sleep(Duration::from_millis(1)).await;
2217 }
2218 })
2219 .await
2220 .unwrap();
2221 assert!(
2222 engine
2223 .produce(
2224 PartitionKey("p".into()),
2225 Bytes::from_static(b"rejected"),
2226 1,
2227 (),
2228 Durability::Buffered
2229 )
2230 .await
2231 .is_err()
2232 );
2233 assert!(engine.flush().await.is_err());
2234 }
2235 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2236 async fn committed_reads_continue_during_new_manifest_publication() {
2237 for fail in [false, true] {
2238 let memory = MemoryBlobStore::new();
2239 let seed_blob: Arc<dyn BlobStore> = Arc::new(memory.clone());
2240 let seed_sequencer = Arc::new(
2241 ManifestSequencer::open(seed_blob.clone(), "manifest/")
2242 .await
2243 .unwrap(),
2244 );
2245 let mut config = FlushConfig::default();
2246 config.max_batches = 1;
2247 config.max_inflight_flushes = 4;
2248 config.linger = Duration::ZERO;
2249 config.budget.enabled = false;
2250 let seed = LogEngine::new(seed_blob, seed_sequencer, config, "data/");
2251 seed.produce(
2252 PartitionKey("p".into()),
2253 Bytes::from_static(b"committed"),
2254 1,
2255 (),
2256 Durability::Sequenced,
2257 )
2258 .await
2259 .unwrap();
2260 drop(seed);
2261 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2262 let (release_tx, release_rx) = oneshot::channel();
2263 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2264 inner: memory,
2265 entered: entered_tx,
2266 release: Mutex::new(Some(release_rx)),
2267 fail,
2268 panic_on_commit: false,
2269 });
2270 let sequencer = Arc::new(
2271 ManifestSequencer::open(blob.clone(), "manifest/")
2272 .await
2273 .unwrap(),
2274 );
2275 let engine = Arc::new(LogEngine::new(
2276 blob.clone(),
2277 sequencer.clone(),
2278 config,
2279 "data/",
2280 ));
2281 let producing = {
2282 let engine = engine.clone();
2283 tokio::spawn(async move {
2284 engine
2285 .produce(
2286 PartitionKey("p".into()),
2287 Bytes::from_static(b"new"),
2288 1,
2289 (),
2290 Durability::Sequenced,
2291 )
2292 .await
2293 })
2294 };
2295 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2296 let mut reading = tokio::task::spawn_blocking(move || {
2297 let p = PartitionKey("p".into());
2298 (
2299 sequencer.high_watermark(&p).unwrap(),
2300 sequencer.lookup(&p, 0).unwrap(),
2301 sequencer.snapshot(),
2302 )
2303 });
2304 let observed = tokio::time::timeout(Duration::from_secs(1), &mut reading).await;
2305 let early_ack = producing.is_finished();
2306 release_tx.send(()).unwrap();
2308 let outcome = producing.await.unwrap();
2309 assert_eq!(outcome.is_err(), fail);
2310 let reads_progressed = observed.is_ok();
2311 let (watermark, entries, snapshot) = match observed {
2312 Ok(result) => result.unwrap(),
2313 Err(_) => reading.await.unwrap(),
2314 };
2315 assert!(!early_ack);
2316 assert!(
2317 reads_progressed,
2318 "durable manifest publication blocked committed index reads"
2319 );
2320 assert_eq!(watermark, 1);
2321 assert_eq!(entries.len(), 1);
2322 assert_eq!(snapshot.manifest_count, 1);
2323 let rows = engine
2324 .fetch(&PartitionKey("p".into()), 0, 100)
2325 .await
2326 .unwrap();
2327 assert_eq!(rows.len(), if fail { 1 } else { 2 });
2328 assert_eq!(rows[0].payload.as_ref(), b"committed");
2329 drop(engine);
2330 let reopened = Arc::new(
2331 ManifestSequencer::open(blob.clone(), "manifest/")
2332 .await
2333 .unwrap(),
2334 );
2335 let engine = LogEngine::new(blob, reopened, config, "data/");
2336 assert_eq!(
2337 engine
2338 .fetch(&PartitionKey("p".into()), 0, 100)
2339 .await
2340 .unwrap()
2341 .len(),
2342 if fail { 1 } else { 2 }
2343 );
2344 }
2345 }
2346 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
2347 async fn manifest_mutations_remain_serialized_while_readers_progress() {
2348 let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1);
2349 let (release_tx, release_rx) = oneshot::channel();
2350 let blob: Arc<dyn BlobStore> = Arc::new(GatedManifestStore {
2351 inner: MemoryBlobStore::new(),
2352 entered: entered_tx,
2353 release: Mutex::new(Some(release_rx)),
2354 fail: false,
2355 panic_on_commit: false,
2356 });
2357 let sequencer = Arc::new(ManifestSequencer::open(blob, "manifest/").await.unwrap());
2358 let commit = |sequencer: Arc<ManifestSequencer>, key: &'static str| {
2359 tokio::task::spawn_blocking(move || {
2360 sequencer.commit(&[CommitBatch {
2361 partition: PartitionKey("p".into()),
2362 record_count: 1,
2363 location: BatchLocation {
2364 object_id: key.into(),
2365 byte_start: 0,
2366 byte_len: 1,
2367 },
2368 meta: &(),
2369 }])
2370 })
2371 };
2372 let first = commit(sequencer.clone(), "data/first");
2373 entered_rx.recv_timeout(Duration::from_secs(5)).unwrap();
2374 let second = commit(sequencer.clone(), "data/second");
2375 let truncating = {
2376 let sequencer = sequencer.clone();
2377 tokio::task::spawn_blocking(move || {
2378 sequencer.truncate_before(&PartitionKey("p".into()), 1)
2379 })
2380 };
2381 let reading = {
2382 let sequencer = sequencer.clone();
2383 tokio::task::spawn_blocking(move || sequencer.high_watermark(&PartitionKey("p".into())))
2384 };
2385 let observed = tokio::time::timeout(Duration::from_secs(1), reading).await;
2386 release_tx.send(()).unwrap();
2387 assert_eq!(
2388 observed.expect("committed reads blocked").unwrap().unwrap(),
2389 0
2390 );
2391 let first = tokio::time::timeout(Duration::from_secs(5), first)
2392 .await
2393 .unwrap()
2394 .unwrap()
2395 .unwrap();
2396 let second = tokio::time::timeout(Duration::from_secs(5), second)
2397 .await
2398 .unwrap()
2399 .unwrap()
2400 .unwrap();
2401 let dropped = tokio::time::timeout(Duration::from_secs(5), truncating)
2402 .await
2403 .unwrap()
2404 .unwrap()
2405 .unwrap();
2406 assert_eq!(
2407 first,
2408 vec![CommitOutcome::Assigned {
2409 base_offset: 0,
2410 record_count: 1
2411 }]
2412 );
2413 assert_eq!(
2414 second,
2415 vec![CommitOutcome::Assigned {
2416 base_offset: 1,
2417 record_count: 1
2418 }]
2419 );
2420 assert_eq!(dropped, vec!["data/first".to_string()]);
2421 let p = PartitionKey("p".into());
2422 assert_eq!(sequencer.high_watermark(&p).unwrap(), 2);
2423 assert_eq!(sequencer.log_start_offset(&p).unwrap(), 1);
2424 let entries = sequencer.lookup(&p, 0).unwrap();
2425 assert_eq!(entries.len(), 1);
2426 assert_eq!(entries[0].location.object_id, "data/second");
2427 assert_eq!(sequencer.snapshot().manifest_count, 2);
2428 }
2429}
2430
2431#[cfg(test)]
2432mod idle_batch_wait_tests {
2433 use super::*;
2434 use std::sync::mpsc;
2435
2436 fn empty_shared(config: FlushConfig) -> Arc<Shared<()>> {
2437 Arc::new(Shared {
2438 queue: Mutex::new(Queue {
2439 items: VecDeque::new(),
2440 bytes: 0,
2441 bytes_in_use: 0,
2442 shutdown: false,
2443 next_seq: 1,
2444 completed_through: 0,
2445 first_failed: None,
2446 force_flush: false,
2447 flush_waiters: Vec::new(),
2448 last_enqueue: None,
2449 oldest_enqueue: None,
2450 }),
2451 cv: Condvar::new(),
2452 max_buffered_bytes: config.max_buffered_bytes.max(config.max_bytes),
2453 budget: Mutex::new(BudgetRuntime::new(config.budget)),
2454 flush_config: config,
2455 })
2456 }
2457
2458 #[test]
2459 fn idle_wait_has_no_periodic_empty_return_and_wakes_for_enqueue_or_shutdown() {
2460 for linger in [Duration::ZERO, Duration::from_millis(1)] {
2461 for shutdown in [false, true] {
2462 let config = FlushConfig {
2463 linger,
2464 ..FlushConfig::default()
2465 };
2466 let shared = empty_shared(config);
2467 let (started_tx, started_rx) = mpsc::channel();
2468 let (result_tx, result_rx) = mpsc::channel();
2469 let worker_shared = shared.clone();
2470 let worker = std::thread::spawn(move || {
2471 started_tx.send(()).unwrap();
2472 assert!(
2473 result_tx
2474 .send(take_batch(&worker_shared, config, true))
2475 .is_ok()
2476 );
2477 });
2478 started_rx.recv_timeout(Duration::from_secs(2)).unwrap();
2479 let early = result_rx.recv_timeout(Duration::from_millis(100));
2480 let returned_while_idle = early.is_ok();
2481 {
2482 let mut q = shared.queue.lock().unwrap();
2483 if shutdown {
2484 q.shutdown = true;
2485 } else {
2486 let payload = Bytes::from_static(b"first-after-idle");
2487 q.bytes = payload.len();
2488 q.bytes_in_use = payload.len();
2489 q.next_seq = 2;
2490 q.last_enqueue = Some(Instant::now());
2491 q.oldest_enqueue = q.last_enqueue;
2492 q.force_flush = true;
2493 q.items.push_back(Pending {
2494 partition: PartitionKey("p".into()),
2495 record_count: 1,
2496 payload,
2497 meta: (),
2498 durability: Durability::Buffered,
2499 responder: None,
2500 seq: 1,
2501 });
2502 }
2503 shared.cv.notify_all();
2504 }
2505 let result = early
2506 .or_else(|_| result_rx.recv_timeout(Duration::from_secs(2)))
2507 .unwrap();
2508 worker.join().unwrap();
2509 assert!(
2510 !returned_while_idle,
2511 "idle worker polled with linger={linger:?}"
2512 );
2513 match result {
2514 TakeBatch::Shutdown => assert!(shutdown),
2515 TakeBatch::Batch(items) => {
2516 assert!(!shutdown);
2517 assert_eq!(items.len(), 1);
2518 assert_eq!(items[0].payload.as_ref(), b"first-after-idle");
2519 assert_eq!(items[0].seq, 1);
2520 }
2521 TakeBatch::Empty => panic!("idle worker returned without work or shutdown"),
2522 }
2523 }
2524 }
2525 }
2526}