Skip to main content

uqa_storage/mvcc/
view.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Provider-independent reads combine one pinned committed boundary with a fixed private command view.
8
9mod fingerprint;
10mod last;
11mod selected;
12pub use selected::SelectedRecordRead;
13
14use std::cmp::Ordering;
15use std::sync::Arc;
16
17use uqa_core::memory::{BudgetedVec, MemoryReservation};
18
19use crate::read_control::StorageReadControl;
20
21use super::{
22    CommitSequence, PreparedRecordWrite, PrivateRecordSnapshot, RecordVersion, ScannedRecord,
23    SharedRecordValue, VersionResult,
24};
25
26#[derive(Clone, Copy)]
27pub struct BorrowedRecord<'a> {
28    pub revision: Option<CommitSequence>,
29    pub value: Option<&'a [u8]>,
30}
31
32/// Visible revision metadata, including tombstones, without retaining the encoded payload.
33#[derive(Clone, Copy, Debug, PartialEq, Eq)]
34pub struct RecordMetadata {
35    pub revision: Option<CommitSequence>,
36    pub live: bool,
37}
38
39/// Exact identity of a live record on one committed/private view. Compare only for the same physical key; history and undo-branch identities distinguish equal payloads that were replaced.
40#[derive(Clone, Copy, Debug, PartialEq, Eq)]
41pub struct VisibleRecordRevision {
42    database: super::DatabaseId,
43    committed: Option<CommitSequence>,
44    private: Option<super::PrivateRecordRevision>,
45}
46
47impl VisibleRecordRevision {
48    pub(crate) fn committed(self, database: super::DatabaseId) -> Option<CommitSequence> {
49        (self.database == database && self.private.is_none())
50            .then_some(self.committed)
51            .flatten()
52    }
53
54    /// The committed sequence this view includes, whether or not private changes accompany it.
55    pub(crate) fn committed_sequence(self) -> Option<CommitSequence> {
56        self.committed
57    }
58
59    pub(crate) fn database(self) -> super::DatabaseId {
60        self.database
61    }
62
63    pub fn is_private(self) -> bool {
64        self.private.is_some()
65    }
66}
67
68impl From<BorrowedRecord<'_>> for RecordMetadata {
69    fn from(record: BorrowedRecord<'_>) -> Self {
70        Self {
71            revision: record.revision,
72            live: record.value.is_some(),
73        }
74    }
75}
76
77pub type RecordValueVisitor<'a> = dyn FnMut(Option<BorrowedRecord<'_>>) -> VersionResult<()> + 'a;
78pub type RecordPointVisitor<'a> =
79    dyn FnMut(&[u8], Option<BorrowedRecord<'_>>) -> VersionResult<bool> + 'a;
80pub type RecordKeyIterator<'a> = dyn Iterator<Item = VersionResult<BudgetedVec<u8>>> + 'a;
81pub type RecordScanVisitor<'a> = dyn FnMut(&[u8], BorrowedRecord<'_>) -> VersionResult<bool> + 'a;
82pub type RecordKeyVisitor<'a> = dyn FnMut(&[u8], RecordMetadata) -> VersionResult<bool> + 'a;
83
84/// A retained committed view whose lease protects visible versions until this owner is dropped. Provider read windows must finish inside each call, without retaining a physical writer or calling user code.
85pub trait CommittedRecordSnapshot: Send + Sync {
86    fn sequence(&self) -> CommitSequence;
87
88    /// The provider's tombstone-reclamation epoch captured under the same admission as this snapshot. `None` means this provider does not supply reclamation-aware observations. Wrappers must forward the original value, never refresh it from current storage.
89    fn reclamation_epoch(&self) -> Option<u64> {
90        None
91    }
92
93    /// The provider's own snapshot type, which its native read paths may serve from physical projections at the same boundary. Wrappers must forward the original provider snapshot.
94    fn provider_snapshot(&self) -> Option<&dyn std::any::Any> {
95        None
96    }
97
98    /// The value [`VersionedPersistence::commit_monitor_version`](super::VersionedPersistence::commit_monitor_version) returned before this snapshot was captured. While the monitor still returns it, nothing was committed after the capture began, so this snapshot is the latest one. `None` when the provider has no monitor. Wrappers must forward the original value.
99    fn commit_monitor(&self) -> Option<u64> {
100        None
101    }
102
103    /// Take the monitor value another snapshot of this same sequence was captured at. Nothing was committed between the two captures, so this snapshot is the latest one for as long as the monitor returns that value. Returns false when the snapshot cannot change its value, and its holder then keeps the other snapshot instead. Keeping this one keeps what it has already read of its records.
104    fn adopt_commit_monitor(&self, _monitor: u64) -> bool {
105        false
106    }
107
108    /// Read a revision and tombstone marker without materializing its value when the provider supports key-only access.
109    fn metadata(
110        &self,
111        key: &[u8],
112        control: &StorageReadControl,
113    ) -> VersionResult<Option<RecordMetadata>> {
114        let mut found = None;
115        self.visit_value(key, control, &mut |record| {
116            found = record.map(Into::into);
117            Ok(())
118        })?;
119        Ok(found)
120    }
121
122    /// Visit ordered record identities, including tombstones, without fetching values from providers with key-only access.
123    fn visit_keys(
124        &self,
125        prefix: &[u8],
126        after: Option<&[u8]>,
127        limit: usize,
128        control: &StorageReadControl,
129        visit: &mut RecordKeyVisitor<'_>,
130    ) -> VersionResult<()> {
131        self.visit_prefix(prefix, after, limit, control, &mut |key, record| {
132            visit(key, record.into())
133        })
134    }
135
136    /// Visit the greatest visible identity with this prefix strictly before `before`, including a tombstone. Invoke the visitor at most once; providers may seek directly without reading preceding keys or values. The default preserves semantics for providers that only implement forward scans.
137    fn visit_last_key(
138        &self,
139        prefix: &[u8],
140        before: Option<&[u8]>,
141        control: &StorageReadControl,
142        visit: &mut RecordKeyVisitor<'_>,
143    ) -> VersionResult<()> {
144        last::by_scan(self, prefix, before, control, visit)
145    }
146
147    /// Return the newest revision at or before this snapshot, including tombstones. Missing identities return `None`.
148    fn get(
149        &self,
150        key: &[u8],
151        control: &StorageReadControl,
152    ) -> VersionResult<Option<RecordVersion<SharedRecordValue>>>;
153
154    /// Return at most `limit` unique keys in ascending order, strictly after `after`, including tombstones. A short page means that the prefix is exhausted at this snapshot; all pages keep the same boundary.
155    fn scan(
156        &self,
157        prefix: &[u8],
158        after: Option<&[u8]>,
159        limit: usize,
160        control: &StorageReadControl,
161    ) -> VersionResult<BudgetedVec<ScannedRecord>>;
162
163    /// Borrow provider-owned bytes for an internal storage visitor. Visitors must not reenter this snapshot or execute SQL/user callbacks.
164    fn visit_value(
165        &self,
166        key: &[u8],
167        control: &StorageReadControl,
168        visit: &mut RecordValueVisitor<'_>,
169    ) -> VersionResult<()> {
170        let record = self.get(key, control)?;
171        visit(record.as_ref().map(|record| BorrowedRecord {
172            revision: Some(record.sequence()),
173            value: record.value().map(|value| &***value),
174        }))?;
175        control.cancellation().check()?;
176        Ok(())
177    }
178
179    /// Check a live value's encoded size before provider materialization, without changing revision or tombstone visibility. The default rejects unsupported providers rather than materializing through `get`.
180    fn visit_value_bounded(
181        &self,
182        _key: &[u8],
183        _max_bytes: usize,
184        control: &StorageReadControl,
185        _visit: &mut RecordValueVisitor<'_>,
186    ) -> VersionResult<()> {
187        control.check()?;
188        Err(crate::StorageBackendError::Other(
189            "size-bounded committed record reads are not supported".into(),
190        )
191        .into())
192    }
193
194    /// Borrow requested values in input order under one optional provider read window. Key production and visitors are internal and must not reenter storage or invoke user code. Produce one key at a time; a false visitor or an error stops before requesting the next key, including duplicate and missing identities.
195    fn visit_values(
196        &self,
197        keys: &mut RecordKeyIterator<'_>,
198        control: &StorageReadControl,
199        visit: &mut RecordPointVisitor<'_>,
200    ) -> VersionResult<()> {
201        loop {
202            control.check()?;
203            let Some(key) = keys.next() else {
204                return Ok(());
205            };
206            let key = key?;
207            let mut more = true;
208            self.visit_value(&key, control, &mut |record| {
209                more = visit(&key, record)?;
210                Ok(())
211            })?;
212            control.check()?;
213            if !more {
214                return Ok(());
215            }
216        }
217    }
218
219    /// Visit ordered versions until the limit, exhaustion or a visitor returning `false`. Implementations with borrowed pages avoid charging their encoded payloads to the caller's decode allowance.
220    fn visit_prefix(
221        &self,
222        prefix: &[u8],
223        after: Option<&[u8]>,
224        limit: usize,
225        control: &StorageReadControl,
226        visit: &mut RecordScanVisitor<'_>,
227    ) -> VersionResult<()> {
228        control.cancellation().check()?;
229        let mut cursor: Option<BudgetedVec<u8>> = None;
230        let mut remaining = limit;
231        while remaining != 0 {
232            let mut page = self.scan(
233                prefix,
234                cursor.as_deref().or(after),
235                remaining.min(64),
236                control,
237            )?;
238            if page.is_empty() {
239                break;
240            }
241            for record in page.iter() {
242                control.cancellation().check()?;
243                if !visit(
244                    &record.key,
245                    BorrowedRecord {
246                        revision: Some(record.version.sequence()),
247                        value: record.version.value().map(|value| &***value),
248                    },
249                )? {
250                    control.cancellation().check()?;
251                    return Ok(());
252                }
253                remaining -= 1;
254            }
255            cursor = page.pop().map(|record| record.key);
256        }
257        control.cancellation().check()?;
258        Ok(())
259    }
260}
261
262struct RetainedSnapshot<T> {
263    snapshot: T,
264    _memory: MemoryReservation,
265}
266
267/// Charge a provider snapshot's retained metadata before allocating its shared owner. The provider snapshot itself owns any version-retention lease required by its format.
268pub fn retain_record_snapshot<T: CommittedRecordSnapshot + 'static>(
269    snapshot: T,
270    control: &StorageReadControl,
271) -> VersionResult<Arc<dyn CommittedRecordSnapshot>> {
272    control.cancellation().check()?;
273    let memory = control
274        .memory()
275        .reserve(std::mem::size_of::<RetainedSnapshot<T>>())?;
276    Ok(Arc::new(RetainedSnapshot {
277        snapshot,
278        _memory: memory,
279    }))
280}
281
282impl<T: CommittedRecordSnapshot> CommittedRecordSnapshot for RetainedSnapshot<T> {
283    fn sequence(&self) -> CommitSequence {
284        self.snapshot.sequence()
285    }
286    fn reclamation_epoch(&self) -> Option<u64> {
287        self.snapshot.reclamation_epoch()
288    }
289    fn provider_snapshot(&self) -> Option<&dyn std::any::Any> {
290        self.snapshot.provider_snapshot()
291    }
292    fn commit_monitor(&self) -> Option<u64> {
293        self.snapshot.commit_monitor()
294    }
295    fn adopt_commit_monitor(&self, monitor: u64) -> bool {
296        self.snapshot.adopt_commit_monitor(monitor)
297    }
298    fn visit_last_key(
299        &self,
300        prefix: &[u8],
301        before: Option<&[u8]>,
302        control: &StorageReadControl,
303        visit: &mut RecordKeyVisitor<'_>,
304    ) -> VersionResult<()> {
305        self.snapshot.visit_last_key(prefix, before, control, visit)
306    }
307    fn metadata(
308        &self,
309        key: &[u8],
310        control: &StorageReadControl,
311    ) -> VersionResult<Option<RecordMetadata>> {
312        self.snapshot.metadata(key, control)
313    }
314    fn visit_keys(
315        &self,
316        prefix: &[u8],
317        after: Option<&[u8]>,
318        limit: usize,
319        control: &StorageReadControl,
320        visit: &mut RecordKeyVisitor<'_>,
321    ) -> VersionResult<()> {
322        self.snapshot
323            .visit_keys(prefix, after, limit, control, visit)
324    }
325    fn get(
326        &self,
327        key: &[u8],
328        control: &StorageReadControl,
329    ) -> VersionResult<Option<RecordVersion<SharedRecordValue>>> {
330        self.snapshot.get(key, control)
331    }
332    fn scan(
333        &self,
334        prefix: &[u8],
335        after: Option<&[u8]>,
336        limit: usize,
337        control: &StorageReadControl,
338    ) -> VersionResult<BudgetedVec<ScannedRecord>> {
339        self.snapshot.scan(prefix, after, limit, control)
340    }
341    fn visit_value(
342        &self,
343        key: &[u8],
344        control: &StorageReadControl,
345        visit: &mut RecordValueVisitor<'_>,
346    ) -> VersionResult<()> {
347        self.snapshot.visit_value(key, control, visit)
348    }
349    fn visit_values(
350        &self,
351        keys: &mut RecordKeyIterator<'_>,
352        control: &StorageReadControl,
353        visit: &mut RecordPointVisitor<'_>,
354    ) -> VersionResult<()> {
355        self.snapshot.visit_values(keys, control, visit)
356    }
357    fn visit_value_bounded(
358        &self,
359        key: &[u8],
360        max_bytes: usize,
361        control: &StorageReadControl,
362        visit: &mut RecordValueVisitor<'_>,
363    ) -> VersionResult<()> {
364        self.snapshot
365            .visit_value_bounded(key, max_bytes, control, visit)
366    }
367    fn visit_prefix(
368        &self,
369        prefix: &[u8],
370        after: Option<&[u8]>,
371        limit: usize,
372        control: &StorageReadControl,
373        visit: &mut RecordScanVisitor<'_>,
374    ) -> VersionResult<()> {
375        self.snapshot
376            .visit_prefix(prefix, after, limit, control, visit)
377    }
378}
379
380/// Visible payload plus its original committed precondition. Private deletions remain observable as tombstones rather than falling back to the committed value.
381pub struct VisibleRecord {
382    original: Option<CommitSequence>,
383    value: Option<SharedRecordValue>,
384    private: bool,
385}
386
387impl VisibleRecord {
388    pub fn original_revision(&self) -> Option<CommitSequence> {
389        self.original
390    }
391
392    pub fn value(&self) -> Option<&[u8]> {
393        self.value.as_ref().map(|value| &***value)
394    }
395
396    pub fn is_private(&self) -> bool {
397        self.private
398    }
399
400    fn committed(version: RecordVersion<SharedRecordValue>) -> Self {
401        let (sequence, value) = version.into_parts();
402        Self {
403            original: Some(sequence),
404            value,
405            private: false,
406        }
407    }
408
409    fn private(write: &PreparedRecordWrite) -> Self {
410        Self {
411            original: write.expected(),
412            value: write.shared_value(),
413            private: true,
414        }
415    }
416}
417
418pub struct ScannedVisibleRecord {
419    pub key: BudgetedVec<u8>,
420    pub record: VisibleRecord,
421}
422
423/// Retain both boundaries across reads and paging. The transaction owner chooses when a later SQL command acquires a new committed snapshot; this reader never advances either boundary implicitly.
424pub struct MergedRecordSnapshot {
425    committed: Arc<dyn CommittedRecordSnapshot>,
426    private: PrivateRecordSnapshot,
427    serializable: Option<super::SerializableReadContext>,
428}
429
430impl MergedRecordSnapshot {
431    pub(in crate::mvcc) fn retained_source(
432        &self,
433        key: &[u8],
434        control: &StorageReadControl,
435    ) -> VersionResult<Option<Arc<dyn crate::key_value::KeyValueRead + Send + Sync>>> {
436        self.private.retained_source(key, control)
437    }
438
439    /// Select one live record's actual revision without materializing its payload. The owning provider supplies this snapshot's transaction-history identity, distinct from any restored data namespace.
440    pub fn record_revision(
441        &self,
442        database: super::DatabaseId,
443        key: &[u8],
444        control: &StorageReadControl,
445    ) -> VersionResult<Option<VisibleRecordRevision>> {
446        control.check()?;
447        let Some(metadata) = self.metadata(key, control)?.filter(|record| record.live) else {
448            control.check()?;
449            return Ok(None);
450        };
451        let private = self.private_keys(key, None, 1, control)?;
452        let private = private
453            .first()
454            .filter(|record| record.key() == key)
455            .map(super::PrivateRecordKey::revision);
456        control.check()?;
457        Ok(Some(VisibleRecordRevision {
458            database,
459            committed: metadata.revision,
460            private,
461        }))
462    }
463
464    /// Share the committed owner and retain the original private revision under its existing allowance.
465    pub fn try_clone(&self) -> VersionResult<Self> {
466        Ok(Self {
467            committed: Arc::clone(&self.committed),
468            private: self.private.try_clone()?,
469            serializable: self.serializable.clone(),
470        })
471    }
472
473    pub fn new(
474        committed: Arc<dyn CommittedRecordSnapshot>,
475        private: PrivateRecordSnapshot,
476    ) -> Self {
477        Self {
478            committed,
479            private,
480            serializable: None,
481        }
482    }
483
484    pub(crate) fn with_serializable(
485        mut self,
486        context: Option<super::SerializableReadContext>,
487    ) -> Self {
488        self.serializable = context;
489        self
490    }
491
492    /// Original logical reader attribution. Clones retain the same participant and fixed data boundaries.
493    pub fn serializable(&self) -> Option<&super::SerializableReadContext> {
494        self.serializable.as_ref()
495    }
496
497    pub fn sequence(&self) -> CommitSequence {
498        self.committed.sequence()
499    }
500
501    /// Whole private command-root identity, restored together with its records on undo.
502    pub fn private_revision(&self) -> Option<super::PrivateRecordRevision> {
503        self.private.revision()
504    }
505
506    /// The pinned committed boundary underlying this command view, without private replacements.
507    pub fn committed(&self) -> &dyn CommittedRecordSnapshot {
508        self.committed.as_ref()
509    }
510
511    pub(super) fn retain_committed(&self) -> Arc<dyn CommittedRecordSnapshot> {
512        Arc::clone(&self.committed)
513    }
514
515    /// Bounded private key metadata for provider-derived cache invalidation; values are never materialized.
516    pub fn private_keys(
517        &self,
518        prefix: &[u8],
519        after: Option<&[u8]>,
520        limit: usize,
521        control: &StorageReadControl,
522    ) -> VersionResult<BudgetedVec<super::PrivateRecordKey>> {
523        self.private.scan_keys(prefix, after, limit, control)
524    }
525
526    /// Newest private revisions per provider-selected cache scope, or exact changed keys when the provider supplies no classifier. The summary shares this view's undo boundary and spills under the same allowance.
527    pub fn private_revision_scopes(
528        &self,
529        after: Option<&[u8]>,
530        limit: usize,
531        control: &StorageReadControl,
532    ) -> VersionResult<BudgetedVec<super::PrivateRecordKey>> {
533        self.private.revision_scopes(after, limit, control)
534    }
535
536    /// Stream private entries on this command boundary without repeated key lookups. Providers can merge their committed rows with each entry's metadata or requested payload.
537    pub fn private_cursor(
538        &self,
539        prefix: &[u8],
540        after: Option<&[u8]>,
541        control: &StorageReadControl,
542    ) -> VersionResult<super::PrivateRecordCursor<'_>> {
543        self.private.cursor(prefix, after, control)
544    }
545
546    pub fn visit_value(
547        &self,
548        key: &[u8],
549        control: &StorageReadControl,
550        visit: &mut RecordValueVisitor<'_>,
551    ) -> VersionResult<()> {
552        if let Some(write) = self.private.get(key, control)? {
553            visit(Some(BorrowedRecord {
554                revision: write.expected(),
555                value: write.value(),
556            }))?;
557            control.cancellation().check()?;
558            return Ok(());
559        }
560        self.committed.visit_value(key, control, visit)
561    }
562
563    /// Keep private replacements ahead of committed values while allowing an unchanged command view to reuse the provider's point-read window.
564    pub fn visit_values(
565        &self,
566        keys: &mut RecordKeyIterator<'_>,
567        control: &StorageReadControl,
568        visit: &mut RecordPointVisitor<'_>,
569    ) -> VersionResult<()> {
570        self.selected(control).visit_values(keys, control, visit)
571    }
572
573    /// Preserve private replacement/tombstone precedence while enforcing the physical source's encoded-value cap.
574    pub fn visit_value_bounded(
575        &self,
576        key: &[u8],
577        max_bytes: usize,
578        control: &StorageReadControl,
579        visit: &mut RecordValueVisitor<'_>,
580    ) -> VersionResult<()> {
581        control.check()?;
582        if let Some(write) = self.private.get(key, control)? {
583            if let Some(value) = write.value() {
584                control.check_value_size(value.len(), max_bytes)?;
585            }
586            visit(Some(BorrowedRecord {
587                revision: write.expected(),
588                value: write.value(),
589            }))?;
590            control.check()?;
591            return Ok(());
592        }
593        self.committed
594            .visit_value_bounded(key, max_bytes, control, visit)
595    }
596
597    pub fn visit_prefix(
598        &self,
599        prefix: &[u8],
600        after: Option<&[u8]>,
601        limit: usize,
602        control: &StorageReadControl,
603        visit: &mut RecordScanVisitor<'_>,
604    ) -> VersionResult<()> {
605        self.private.visit_merged::<super::projection::Values>(
606            &*self.committed,
607            prefix,
608            after,
609            limit,
610            control,
611            visit,
612        )
613    }
614
615    pub fn metadata(
616        &self,
617        key: &[u8],
618        control: &StorageReadControl,
619    ) -> VersionResult<Option<RecordMetadata>> {
620        if let Some(metadata) = self.private.metadata(key, control)? {
621            return Ok(Some(metadata));
622        }
623        self.committed.metadata(key, control)
624    }
625
626    pub fn visit_keys(
627        &self,
628        prefix: &[u8],
629        after: Option<&[u8]>,
630        limit: usize,
631        control: &StorageReadControl,
632        visit: &mut RecordKeyVisitor<'_>,
633    ) -> VersionResult<()> {
634        self.private.visit_merged::<super::projection::Keys>(
635            &*self.committed,
636            prefix,
637            after,
638            limit,
639            control,
640            visit,
641        )
642    }
643
644    /// Whether this view holds a private change of `key`, whose committed record is then not what a read of the key returns.
645    pub fn has_private_change(
646        &self,
647        key: &[u8],
648        control: &StorageReadControl,
649    ) -> VersionResult<bool> {
650        Ok(self.private.get(key, control)?.is_some())
651    }
652
653    pub fn get(
654        &self,
655        key: &[u8],
656        control: &StorageReadControl,
657    ) -> VersionResult<Option<VisibleRecord>> {
658        if let Some(write) = self.private.get(key, control)? {
659            return Ok(Some(VisibleRecord::private(&write)));
660        }
661        Ok(self
662            .committed
663            .get(key, control)?
664            .map(VisibleRecord::committed))
665    }
666
667    /// Merge bounded pages without duplicate identities or payload copies. Tombstones are returned so callers can preserve revision observations even when no row is visible.
668    pub fn scan(
669        &self,
670        prefix: &[u8],
671        after: Option<&[u8]>,
672        limit: usize,
673        control: &StorageReadControl,
674    ) -> VersionResult<BudgetedVec<ScannedVisibleRecord>> {
675        control.cancellation().check()?;
676        let mut result = BudgetedVec::new(control.memory());
677        if limit == 0 {
678            return Ok(result);
679        }
680        let (committed, _committed_memory) = self
681            .committed
682            .scan(prefix, after, limit, control)?
683            .into_parts();
684        let (private, _private_memory) = self
685            .private
686            .scan(prefix, after, limit, control)?
687            .into_parts();
688        let mut committed = committed.into_iter().peekable();
689        let mut private = private.into_iter().peekable();
690        while result.len() < limit {
691            control.cancellation().check()?;
692            let order = match (committed.peek(), private.peek()) {
693                (Some(committed), Some(private)) => committed.key.as_ref().cmp(private.key()),
694                (Some(_), None) => Ordering::Less,
695                (None, Some(_)) => Ordering::Greater,
696                (None, None) => break,
697            };
698            let record = match order {
699                Ordering::Less => {
700                    let committed = committed.next().expect("peeked committed record");
701                    ScannedVisibleRecord {
702                        key: committed.key,
703                        record: VisibleRecord::committed(committed.version),
704                    }
705                }
706                Ordering::Equal => {
707                    let committed = committed.next().expect("peeked committed record");
708                    let private = private.next().expect("peeked private record");
709                    ScannedVisibleRecord {
710                        key: committed.key,
711                        record: VisibleRecord::private(&private),
712                    }
713                }
714                Ordering::Greater => {
715                    let private = private.next().expect("peeked private record");
716                    let mut key = BudgetedVec::new(control.memory());
717                    key.extend_from_slice(private.key())?;
718                    ScannedVisibleRecord {
719                        key,
720                        record: VisibleRecord::private(&private),
721                    }
722                }
723            };
724            result.push(record)?;
725        }
726        Ok(result)
727    }
728}