pub struct AlignedBoundaryStream { /* private fields */ }Expand description
A stream wrapper that lazily aligns byte boundaries to newline characters.
Given a raw byte stream starting from fetch_start (which is start - 1
for non-zero starts, or 0), this stream:
- Skips bytes until the first newline is found (start alignment)
- Passes through data until the
endboundary is reached - Continues past
endto find the terminating newline (end alignment)
When the initial byte stream is exhausted during step 3 and the file has
not been fully read, ScanningLastTerminator issues additional bounded
get_opts calls (END_SCAN_LOOKAHEAD bytes each) until the newline is
found or EOF is reached.
Implementations§
Source§impl AlignedBoundaryStream
impl AlignedBoundaryStream
Sourcepub async fn new(
store: Arc<dyn ObjectStore>,
location: Path,
raw_start: u64,
raw_end: u64,
file_size: u64,
terminator: u8,
) -> Result<Self>
pub async fn new( store: Arc<dyn ObjectStore>, location: Path, raw_start: u64, raw_end: u64, file_size: u64, terminator: u8, ) -> Result<Self>
Open a ranged byte stream from store and return a ready-to-poll
AlignedBoundaryStream.
Issues a single bounded get_opts call covering
[fetch_start, raw_end + END_SCAN_LOOKAHEAD). If the terminating
newline is not found within that window, ScanningLastTerminator
automatically issues additional END_SCAN_LOOKAHEAD-sized GETs
via store until the newline is found or EOF is reached.
Trait Implementations§
Source§impl Stream for AlignedBoundaryStream
impl Stream for AlignedBoundaryStream
Auto Trait Implementations§
impl !Freeze for AlignedBoundaryStream
impl !RefUnwindSafe for AlignedBoundaryStream
impl !Sync for AlignedBoundaryStream
impl !UnwindSafe for AlignedBoundaryStream
impl Send for AlignedBoundaryStream
impl Unpin for AlignedBoundaryStream
impl UnsafeUnpin for AlignedBoundaryStream
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§impl<St> StreamExt for St
impl<St> StreamExt for St
Source§fn next(&mut self) -> Next<'_, Self>where
Self: Unpin,
fn next(&mut self) -> Next<'_, Self>where
Self: Unpin,
None if the
stream is finished. Read moreSource§fn try_next<T, E>(&mut self) -> TryNext<'_, Self>
fn try_next<T, E>(&mut self) -> TryNext<'_, Self>
Source§fn map<T, F>(self, f: F) -> Map<Self, F>
fn map<T, F>(self, f: F) -> Map<Self, F>
Source§fn map_while<T, F>(self, f: F) -> MapWhile<Self, F>
fn map_while<T, F>(self, f: F) -> MapWhile<Self, F>
None. Read moreSource§fn then<F, Fut>(self, f: F) -> Then<Self, Fut, F>
fn then<F, Fut>(self, f: F) -> Then<Self, Fut, F>
Source§fn merge<U>(self, other: U) -> Merge<Self, U>
fn merge<U>(self, other: U) -> Merge<Self, U>
Source§fn filter<F>(self, f: F) -> Filter<Self, F>
fn filter<F>(self, f: F) -> Filter<Self, F>
Source§fn filter_map<T, F>(self, f: F) -> FilterMap<Self, F>
fn filter_map<T, F>(self, f: F) -> FilterMap<Self, F>
Source§fn fuse(self) -> Fuse<Self>where
Self: Sized,
fn fuse(self) -> Fuse<Self>where
Self: Sized,
None. Read moreSource§fn take(self, n: usize) -> Take<Self>where
Self: Sized,
fn take(self, n: usize) -> Take<Self>where
Self: Sized,
n items of the underlying stream. Read moreSource§fn take_while<F>(self, f: F) -> TakeWhile<Self, F>
fn take_while<F>(self, f: F) -> TakeWhile<Self, F>
true. Read moreSource§fn skip(self, n: usize) -> Skip<Self>where
Self: Sized,
fn skip(self, n: usize) -> Skip<Self>where
Self: Sized,
n first items of the
underlying stream. Read moreSource§fn skip_while<F>(self, f: F) -> SkipWhile<Self, F>
fn skip_while<F>(self, f: F) -> SkipWhile<Self, F>
true. Read moreSource§fn all<F>(&mut self, f: F) -> AllFuture<'_, Self, F>
fn all<F>(&mut self, f: F) -> AllFuture<'_, Self, F>
Source§fn any<F>(&mut self, f: F) -> AnyFuture<'_, Self, F>
fn any<F>(&mut self, f: F) -> AnyFuture<'_, Self, F>
Source§fn chain<U>(self, other: U) -> Chain<Self, U>
fn chain<U>(self, other: U) -> Chain<Self, U>
Source§fn fold<B, F>(self, init: B, f: F) -> FoldFuture<Self, B, F>
fn fold<B, F>(self, init: B, f: F) -> FoldFuture<Self, B, F>
Source§fn collect<T>(
self,
) -> Collect<Self, T, <T as FromStreamPriv<Self::Item>>::InternalCollection>
fn collect<T>( self, ) -> Collect<Self, T, <T as FromStreamPriv<Self::Item>>::InternalCollection>
Source§fn timeout(self, duration: Duration) -> Timeout<Self>where
Self: Sized,
fn timeout(self, duration: Duration) -> Timeout<Self>where
Self: Sized,
Source§fn timeout_repeating(self, interval: Interval) -> TimeoutRepeating<Self>where
Self: Sized,
fn timeout_repeating(self, interval: Interval) -> TimeoutRepeating<Self>where
Self: Sized,
Source§fn throttle(self, duration: Duration) -> Throttle<Self>where
Self: Sized,
fn throttle(self, duration: Duration) -> Throttle<Self>where
Self: Sized,
Source§fn chunks_timeout(
self,
max_size: usize,
duration: Duration,
) -> ChunksTimeout<Self>where
Self: Sized,
fn chunks_timeout(
self,
max_size: usize,
duration: Duration,
) -> ChunksTimeout<Self>where
Self: Sized,
Source§impl<T> StreamExt for T
impl<T> StreamExt for T
Source§fn next(&mut self) -> Next<'_, Self>where
Self: Unpin,
fn next(&mut self) -> Next<'_, Self>where
Self: Unpin,
Source§fn into_future(self) -> StreamFuture<Self>
fn into_future(self) -> StreamFuture<Self>
Source§fn map<T, F>(self, f: F) -> Map<Self, F>
fn map<T, F>(self, f: F) -> Map<Self, F>
Source§fn enumerate(self) -> Enumerate<Self>where
Self: Sized,
fn enumerate(self) -> Enumerate<Self>where
Self: Sized,
Source§fn filter<Fut, F>(self, f: F) -> Filter<Self, Fut, F>
fn filter<Fut, F>(self, f: F) -> Filter<Self, Fut, F>
Source§fn filter_map<Fut, T, F>(self, f: F) -> FilterMap<Self, Fut, F>
fn filter_map<Fut, T, F>(self, f: F) -> FilterMap<Self, Fut, F>
Source§fn then<Fut, F>(self, f: F) -> Then<Self, Fut, F>
fn then<Fut, F>(self, f: F) -> Then<Self, Fut, F>
Source§fn collect<C>(self) -> Collect<Self, C>
fn collect<C>(self) -> Collect<Self, C>
Source§fn unzip<A, B, FromA, FromB>(self) -> Unzip<Self, FromA, FromB>
fn unzip<A, B, FromA, FromB>(self) -> Unzip<Self, FromA, FromB>
Source§fn concat(self) -> Concat<Self>
fn concat(self) -> Concat<Self>
Source§fn count(self) -> Count<Self>where
Self: Sized,
fn count(self) -> Count<Self>where
Self: Sized,
Source§fn fold<T, Fut, F>(self, init: T, f: F) -> Fold<Self, Fut, T, F>
fn fold<T, Fut, F>(self, init: T, f: F) -> Fold<Self, Fut, T, F>
Source§fn any<Fut, F>(self, f: F) -> Any<Self, Fut, F>
fn any<Fut, F>(self, f: F) -> Any<Self, Fut, F>
true if any element in stream satisfied a predicate. Read moreSource§fn all<Fut, F>(self, f: F) -> All<Self, Fut, F>
fn all<Fut, F>(self, f: F) -> All<Self, Fut, F>
true if all element in stream satisfied a predicate. Read moreSource§fn flatten(self) -> Flatten<Self>
fn flatten(self) -> Flatten<Self>
Source§fn flatten_unordered(
self,
limit: impl Into<Option<usize>>,
) -> FlattenUnorderedWithFlowController<Self, ()>
fn flatten_unordered( self, limit: impl Into<Option<usize>>, ) -> FlattenUnorderedWithFlowController<Self, ()>
Source§fn flat_map_unordered<U, F>(
self,
limit: impl Into<Option<usize>>,
f: F,
) -> FlatMapUnordered<Self, U, F>
fn flat_map_unordered<U, F>( self, limit: impl Into<Option<usize>>, f: F, ) -> FlatMapUnordered<Self, U, F>
StreamExt::map but flattens nested Streams
and polls them concurrently, yielding items in any order, as they made
available. Read moreSource§fn scan<S, B, Fut, F>(self, initial_state: S, f: F) -> Scan<Self, S, Fut, F>
fn scan<S, B, Fut, F>(self, initial_state: S, f: F) -> Scan<Self, S, Fut, F>
StreamExt::fold that holds internal state
and produces a new stream. Read moreSource§fn skip_while<Fut, F>(self, f: F) -> SkipWhile<Self, Fut, F>
fn skip_while<Fut, F>(self, f: F) -> SkipWhile<Self, Fut, F>
true. Read moreSource§fn take_while<Fut, F>(self, f: F) -> TakeWhile<Self, Fut, F>
fn take_while<Fut, F>(self, f: F) -> TakeWhile<Self, Fut, F>
true. Read moreSource§fn take_until<Fut>(self, fut: Fut) -> TakeUntil<Self, Fut>
fn take_until<Fut>(self, fut: Fut) -> TakeUntil<Self, Fut>
Source§fn for_each<Fut, F>(self, f: F) -> ForEach<Self, Fut, F>
fn for_each<Fut, F>(self, f: F) -> ForEach<Self, Fut, F>
Source§fn for_each_concurrent<Fut, F>(
self,
limit: impl Into<Option<usize>>,
f: F,
) -> ForEachConcurrent<Self, Fut, F>
fn for_each_concurrent<Fut, F>( self, limit: impl Into<Option<usize>>, f: F, ) -> ForEachConcurrent<Self, Fut, F>
Source§fn take(self, n: usize) -> Take<Self>where
Self: Sized,
fn take(self, n: usize) -> Take<Self>where
Self: Sized,
n items of the underlying stream. Read moreSource§fn skip(self, n: usize) -> Skip<Self>where
Self: Sized,
fn skip(self, n: usize) -> Skip<Self>where
Self: Sized,
n items of the underlying stream. Read moreSource§fn catch_unwind(self) -> CatchUnwind<Self>where
Self: Sized + UnwindSafe,
fn catch_unwind(self) -> CatchUnwind<Self>where
Self: Sized + UnwindSafe,
Source§fn boxed<'a>(self) -> Pin<Box<dyn Stream<Item = Self::Item> + Send + 'a>>
fn boxed<'a>(self) -> Pin<Box<dyn Stream<Item = Self::Item> + Send + 'a>>
Source§fn boxed_local<'a>(self) -> Pin<Box<dyn Stream<Item = Self::Item> + 'a>>where
Self: Sized + 'a,
fn boxed_local<'a>(self) -> Pin<Box<dyn Stream<Item = Self::Item> + 'a>>where
Self: Sized + 'a,
Source§fn buffered(self, n: usize) -> Buffered<Self>
fn buffered(self, n: usize) -> Buffered<Self>
Source§fn buffer_unordered(self, n: usize) -> BufferUnordered<Self>
fn buffer_unordered(self, n: usize) -> BufferUnordered<Self>
Source§fn zip<St>(self, other: St) -> Zip<Self, St>
fn zip<St>(self, other: St) -> Zip<Self, St>
Source§fn peekable(self) -> Peekable<Self>where
Self: Sized,
fn peekable(self) -> Peekable<Self>where
Self: Sized,
peek method. Read moreSource§fn chunks(self, capacity: usize) -> Chunks<Self>where
Self: Sized,
fn chunks(self, capacity: usize) -> Chunks<Self>where
Self: Sized,
Source§fn ready_chunks(self, capacity: usize) -> ReadyChunks<Self>where
Self: Sized,
fn ready_chunks(self, capacity: usize) -> ReadyChunks<Self>where
Self: Sized,
Source§fn forward<S>(self, sink: S) -> Forward<Self, S>
fn forward<S>(self, sink: S) -> Forward<Self, S>
Source§fn split<Item>(self) -> (SplitSink<Self, Item>, SplitStream<Self>)
fn split<Item>(self) -> (SplitSink<Self, Item>, SplitStream<Self>)
Source§fn inspect<F>(self, f: F) -> Inspect<Self, F>
fn inspect<F>(self, f: F) -> Inspect<Self, F>
Source§fn left_stream<B>(self) -> Either<Self, B>
fn left_stream<B>(self) -> Either<Self, B>
Source§fn right_stream<B>(self) -> Either<B, Self>
fn right_stream<B>(self) -> Either<B, Self>
Source§fn poll_next_unpin(&mut self, cx: &mut Context<'_>) -> Poll<Option<Self::Item>>where
Self: Unpin,
fn poll_next_unpin(&mut self, cx: &mut Context<'_>) -> Poll<Option<Self::Item>>where
Self: Unpin,
Stream::poll_next on Unpin
stream types.Source§fn select_next_some(&mut self) -> SelectNextSome<'_, Self>where
Self: Unpin + FusedStream,
fn select_next_some(&mut self) -> SelectNextSome<'_, Self>where
Self: Unpin + FusedStream,
Source§impl<S> TryStreamExt for S
impl<S> TryStreamExt for S
Source§fn err_into<E>(self) -> ErrInto<Self, E>
fn err_into<E>(self) -> ErrInto<Self, E>
Source§fn map_ok<T, F>(self, f: F) -> MapOk<Self, F>
fn map_ok<T, F>(self, f: F) -> MapOk<Self, F>
Source§fn map_err<E, F>(self, f: F) -> MapErr<Self, F>
fn map_err<E, F>(self, f: F) -> MapErr<Self, F>
Source§fn and_then<Fut, F>(self, f: F) -> AndThen<Self, Fut, F>
fn and_then<Fut, F>(self, f: F) -> AndThen<Self, Fut, F>
f. Read moreSource§fn or_else<Fut, F>(self, f: F) -> OrElse<Self, Fut, F>
fn or_else<Fut, F>(self, f: F) -> OrElse<Self, Fut, F>
f. Read moreSource§fn inspect_ok<F>(self, f: F) -> InspectOk<Self, F>
fn inspect_ok<F>(self, f: F) -> InspectOk<Self, F>
Source§fn inspect_err<F>(self, f: F) -> InspectErr<Self, F>
fn inspect_err<F>(self, f: F) -> InspectErr<Self, F>
Source§fn into_stream(self) -> IntoStream<Self>where
Self: Sized,
fn into_stream(self) -> IntoStream<Self>where
Self: Sized,
Source§fn try_next(&mut self) -> TryNext<'_, Self>where
Self: Unpin,
fn try_next(&mut self) -> TryNext<'_, Self>where
Self: Unpin,
Source§fn try_for_each<Fut, F>(self, f: F) -> TryForEach<Self, Fut, F>
fn try_for_each<Fut, F>(self, f: F) -> TryForEach<Self, Fut, F>
Source§fn try_skip_while<Fut, F>(self, f: F) -> TrySkipWhile<Self, Fut, F>
fn try_skip_while<Fut, F>(self, f: F) -> TrySkipWhile<Self, Fut, F>
true. Read moreSource§fn try_take_while<Fut, F>(self, f: F) -> TryTakeWhile<Self, Fut, F>
fn try_take_while<Fut, F>(self, f: F) -> TryTakeWhile<Self, Fut, F>
true. Read moreSource§fn try_for_each_concurrent<Fut, F>(
self,
limit: impl Into<Option<usize>>,
f: F,
) -> TryForEachConcurrent<Self, Fut, F>
fn try_for_each_concurrent<Fut, F>( self, limit: impl Into<Option<usize>>, f: F, ) -> TryForEachConcurrent<Self, Fut, F>
Source§fn try_collect<C>(self) -> TryCollect<Self, C>
fn try_collect<C>(self) -> TryCollect<Self, C>
Source§fn try_chunks(self, capacity: usize) -> TryChunks<Self>where
Self: Sized,
fn try_chunks(self, capacity: usize) -> TryChunks<Self>where
Self: Sized,
Source§fn try_ready_chunks(self, capacity: usize) -> TryReadyChunks<Self>where
Self: Sized,
fn try_ready_chunks(self, capacity: usize) -> TryReadyChunks<Self>where
Self: Sized,
Source§fn try_filter<Fut, F>(self, f: F) -> TryFilter<Self, Fut, F>
fn try_filter<Fut, F>(self, f: F) -> TryFilter<Self, Fut, F>
Source§fn try_filter_map<Fut, F, T>(self, f: F) -> TryFilterMap<Self, Fut, F>
fn try_filter_map<Fut, F, T>(self, f: F) -> TryFilterMap<Self, Fut, F>
Source§fn try_flatten_unordered(
self,
limit: impl Into<Option<usize>>,
) -> TryFlattenUnordered<Self>
fn try_flatten_unordered( self, limit: impl Into<Option<usize>>, ) -> TryFlattenUnordered<Self>
Source§fn try_flatten(self) -> TryFlatten<Self>
fn try_flatten(self) -> TryFlatten<Self>
Source§fn try_fold<T, Fut, F>(self, init: T, f: F) -> TryFold<Self, Fut, T, F>
fn try_fold<T, Fut, F>(self, init: T, f: F) -> TryFold<Self, Fut, T, F>
Source§fn try_concat(self) -> TryConcat<Self>
fn try_concat(self) -> TryConcat<Self>
Source§fn try_buffer_unordered(self, n: usize) -> TryBufferUnordered<Self>
fn try_buffer_unordered(self, n: usize) -> TryBufferUnordered<Self>
Source§fn try_buffered(self, n: usize) -> TryBuffered<Self>
fn try_buffered(self, n: usize) -> TryBuffered<Self>
Source§fn try_poll_next_unpin(
&mut self,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Self::Ok, Self::Error>>>where
Self: Unpin,
fn try_poll_next_unpin(
&mut self,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Self::Ok, Self::Error>>>where
Self: Unpin,
TryStream::try_poll_next on Unpin
stream types.Source§fn into_async_read(self) -> IntoAsyncRead<Self>
fn into_async_read(self) -> IntoAsyncRead<Self>
AsyncBufRead. Read moreSource§fn try_all<Fut, F>(self, f: F) -> TryAll<Self, Fut, F>
fn try_all<Fut, F>(self, f: F) -> TryAll<Self, Fut, F>
Err is encountered or if an Ok item is found
that does not satisfy the predicate. Read more