Skip to main content

rig_http/http_client/
framing.rs

1//! Incremental SSE and newline-delimited framing of response bytes.
2//! SSE dispatch requires a blank line; incomplete events are not flushed at EOF.
3//! NDJSON permits a final unterminated line through [`NdjsonFramer::finish`].
4//! A framer holds at most its limit ([`FRAME_LIMIT`] unless given another)
5//! plus one byte of a frame: a larger frame is refused as [`TooLarge`].
6//!
7//! ```
8//! use rig_http::http_client::framing::NdjsonFramer;
9//!
10//! let mut framer = NdjsonFramer::new();
11//! assert_eq!(framer.push(b"{}\n").collect::<Vec<_>>(), vec![Ok(b"{}".to_vec())]);
12//! ```
13
14use std::borrow::Cow;
15
16/// The most bytes one frame of a streamed reply may hold: an SSE event's
17/// lines since the last blank line that a framer keeps (its data and any
18/// unfinished line), or an NDJSON line. Far above any real frame, and
19/// below what would exhaust memory.
20pub const FRAME_LIMIT: usize = 64 * 1024 * 1024;
21
22/// A frame, line or input larger than its reader's limit, refused before
23/// the reader's buffer grew past it.
24#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
25#[error("more than {limit} bytes")]
26pub struct TooLarge {
27    /// The most bytes the reader takes.
28    pub limit: usize,
29    /// The bytes it held when it refused: past `limit`, and the whole size
30    /// only when the input ended there.
31    pub asked: usize,
32}
33
34/// How a reply's bytes are split into wire frames.
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
36pub enum Framing {
37    /// `text/event-stream`: WHATWG event-stream grammar ([`SseFramer`]).
38    Sse,
39    /// Newline-delimited JSON ([`NdjsonFramer`]).
40    Ndjson,
41    /// The whole body is one frame.
42    Whole,
43}
44
45impl Framing {
46    /// The incremental framer of a streamed reply. A whole body is one
47    /// frame, so it has none.
48    pub fn framer(self) -> Option<Framer> {
49        self.framer_with_limit(FRAME_LIMIT)
50    }
51
52    /// The incremental framer of a streamed reply, refusing a frame past
53    /// `limit`.
54    fn framer_with_limit(self, limit: usize) -> Option<Framer> {
55        let lines = match self {
56            Self::Sse => Lines::Sse(SseFramer::with_limit(limit)),
57            Self::Ndjson => Lines::Ndjson(NdjsonFramer::with_limit(limit)),
58            Self::Whole => return None,
59        };
60        Some(Framer { lines })
61    }
62
63    /// Split a whole reply body into the frames a decoder reads: one per SSE
64    /// event with a non-blank `data` field, one per NDJSON line, or the
65    /// whole body. The body is in memory already, so no frame of it is
66    /// refused for its size.
67    ///
68    /// ```
69    /// use rig_http::http_client::framing::{Framing, WireFrame};
70    ///
71    /// let frames = Framing::Sse.split(b"data: {}\n\ndata: [DONE]\n\n");
72    /// assert_eq!(frames.len(), 2);
73    /// assert_eq!(frames[1].as_str(), "[DONE]");
74    /// ```
75    pub fn split(&self, bytes: &[u8]) -> Vec<WireFrame> {
76        let Some(mut framer) = self.framer_with_limit(usize::MAX) else {
77            return (!bytes.is_empty())
78                .then(|| WireFrame::from_bytes(bytes.to_vec()))
79                .into_iter()
80                .collect();
81        };
82        let mut frames: Vec<WireFrame> = framer.push(bytes).flatten().collect();
83        frames.extend(framer.finish());
84        frames
85    }
86}
87
88/// One transport frame, after framing but before decoding.
89///
90/// The transport layer (SSE framer, NDJSON splitter, websocket reader) owns
91/// byte splitting and yields these; a decoder never splits bytes.
92#[derive(Debug, Clone, PartialEq, Eq)]
93pub enum WireFrame {
94    /// Text payload, such as an SSE `data:` field or WebSocket message.
95    Text(String),
96    /// Byte payload, such as an NDJSON line or binary frame.
97    Bytes(Vec<u8>),
98}
99
100impl WireFrame {
101    /// The frame payload as text (lossy for byte frames).
102    pub fn as_str(&self) -> Cow<'_, str> {
103        match self {
104            Self::Text(text) => Cow::Borrowed(text),
105            Self::Bytes(bytes) => String::from_utf8_lossy(bytes),
106        }
107    }
108
109    /// The frame payload's bytes.
110    pub fn as_bytes(&self) -> &[u8] {
111        match self {
112            Self::Text(text) => text.as_bytes(),
113            Self::Bytes(bytes) => bytes,
114        }
115    }
116
117    /// The frame payload's bytes, owned.
118    pub fn into_bytes(self) -> Vec<u8> {
119        match self {
120            Self::Text(text) => text.into_bytes(),
121            Self::Bytes(bytes) => bytes,
122        }
123    }
124
125    /// A frame of `bytes`: text when they are UTF-8.
126    pub fn from_bytes(bytes: Vec<u8>) -> Self {
127        match String::from_utf8(bytes) {
128            Ok(text) => Self::Text(text),
129            Err(error) => Self::Bytes(error.into_bytes()),
130        }
131    }
132}
133
134/// The incremental framer of one streamed reply, from [`Framing::framer`].
135#[derive(Debug)]
136pub struct Framer {
137    lines: Lines,
138}
139
140#[derive(Debug)]
141enum Lines {
142    Sse(SseFramer),
143    Ndjson(NdjsonFramer),
144}
145
146impl Framer {
147    /// Feed one transport chunk; yields every frame it completed, and
148    /// [`TooLarge`] in the place of a frame past the limit: SSE yields
149    /// nothing after it, NDJSON reads on after that line. An SSE event whose
150    /// data is only whitespace is a heartbeat, not a frame.
151    pub fn push(&mut self, chunk: &[u8]) -> impl Iterator<Item = Result<WireFrame, TooLarge>> + '_ {
152        let (sse, ndjson) = match &mut self.lines {
153            Lines::Sse(framer) => (Some(framer.push(chunk)), None),
154            Lines::Ndjson(framer) => (None, Some(framer.push(chunk))),
155        };
156        let events = sse.into_iter().flatten();
157        let events = events.filter(|event| {
158            event
159                .as_ref()
160                .map_or(true, |event| !event.data.trim().is_empty())
161        });
162        let events = events.map(|event| event.map(|event| WireFrame::Text(event.data)));
163        let lines = ndjson.into_iter().flatten();
164        events.chain(lines.map(|line| line.map(WireFrame::from_bytes)))
165    }
166
167    /// The stream ended: NDJSON's unterminated last line. SSE dispatches
168    /// only on a blank line, so an unterminated event is never a frame.
169    pub fn finish(&mut self) -> Option<WireFrame> {
170        match &mut self.lines {
171            Lines::Sse(_) => None,
172            Lines::Ndjson(framer) => framer.finish().map(WireFrame::from_bytes),
173        }
174    }
175
176    /// Bytes pushed but not yet part of a frame, for truncation
177    /// diagnostics: [`SseFramer::pending`] or [`NdjsonFramer::pending`].
178    pub fn pending(&self) -> usize {
179        match &self.lines {
180            Lines::Sse(framer) => framer.pending(),
181            Lines::Ndjson(framer) => framer.pending(),
182        }
183    }
184}
185
186/// One dispatched `text/event-stream` event.
187#[derive(Debug, Clone, Default, PartialEq, Eq)]
188pub struct SseEvent {
189    /// The `event:` field, `"message"` when the stream named none.
190    pub event: Cow<'static, str>,
191    /// The joined `data:` fields, without the terminating newline.
192    pub data: String,
193    /// The stream's last event id, empty when none was ever set.
194    pub id: String,
195    /// Most recent parsed `retry:` value since the previous emitted event,
196    /// saturating at `u64::MAX`. This value does not trigger automatic
197    /// reconnection.
198    pub retry: Option<u64>,
199}
200
201/// A field recognized by the SSE grammar.
202#[derive(Debug, Clone, Copy, PartialEq, Eq)]
203pub enum SseField {
204    /// A payload line.
205    Data,
206    /// An event type.
207    Event,
208    /// An accepted event id.
209    Id,
210    /// A blank dispatch line.
211    Blank,
212    /// A comment, retry, ignored id or unknown field.
213    Other,
214}
215
216/// Original byte ranges of one line consumed by [`SseFramer`].
217#[derive(Debug, Clone, PartialEq, Eq)]
218pub struct SseLine {
219    /// Whole line, including its line ending, at saturating stream offsets.
220    pub span: std::ops::Range<usize>,
221    /// Line content without its ending or an initial BOM.
222    pub content: std::ops::Range<usize>,
223    /// Field value after a colon and optional space, when present.
224    pub value: Option<std::ops::Range<usize>>,
225    /// The field accepted by the grammar.
226    pub field: SseField,
227}
228
229/// A parsed event and the byte offset where it became dispatchable.
230#[derive(Debug, Clone, PartialEq, Eq)]
231pub struct RecordedSseEvent {
232    /// The grammar's parsed event.
233    pub event: SseEvent,
234    /// Saturating stream offset of its completing line ending or retained EOF.
235    pub end: usize,
236}
237
238/// Parsed events and source lines consumed by a recording operation.
239#[derive(Debug, Default)]
240pub struct SseBatch {
241    /// Events emitted in order, with their completion offsets.
242    pub events: Vec<RecordedSseEvent>,
243    /// Source lines in order, including non-dispatching lines.
244    pub lines: Vec<SseLine>,
245}
246
247/// The optional leading byte-order mark, which the grammar ignores.
248const BOM: [u8; 3] = [0xef, 0xbb, 0xbf];
249
250/// A push parser for the WHATWG `text/event-stream` grammar, which holds at
251/// most its limit plus one byte of an event ([`TooLarge`]).
252#[derive(Debug)]
253pub struct SseFramer {
254    /// Bytes not yet consumed as a complete line.
255    buffer: Vec<u8>,
256    /// How many bytes at the start of `buffer` hold no line end.
257    scanned: usize,
258    /// The last line ended at a CR that was the last byte pushed, so an LF
259    /// opening the next chunk completes that CRLF.
260    after_cr: bool,
261    /// Events completed by the current `push`, and its refusal.
262    ready: Vec<Result<SseEvent, TooLarge>>,
263    event_type: String,
264    data: String,
265    last_event_id: String,
266    retry: Option<u64>,
267    /// How many BOM bytes have matched so far, while the prefix is undecided.
268    bom_prefix: usize,
269    bom_done: bool,
270    /// Bytes of complete lines consumed since the last blank line.
271    since_dispatch: usize,
272    /// Absolute byte position after the last consumed line or BOM.
273    offset: usize,
274    /// The most bytes of an event it holds: its unfinished line and its data.
275    limit: usize,
276    /// The bytes of the ids copied into the events of the current push.
277    copied: usize,
278    /// Whether it refused an event, after which it reads nothing more.
279    refused: bool,
280    /// Whether it parses a recording: text in memory already, which no
281    /// limit applies to.
282    recording: bool,
283}
284
285impl Default for SseFramer {
286    fn default() -> Self {
287        Self::with_limit(FRAME_LIMIT)
288    }
289}
290
291impl SseFramer {
292    /// A framer at the start of a stream, refusing an event past
293    /// [`FRAME_LIMIT`].
294    pub fn new() -> Self {
295        Self::default()
296    }
297
298    /// A framer at the start of a stream, refusing an event that holds more
299    /// than `limit` bytes.
300    pub fn with_limit(limit: usize) -> Self {
301        Self {
302            buffer: Vec::new(),
303            scanned: 0,
304            after_cr: false,
305            ready: Vec::new(),
306            event_type: String::new(),
307            data: String::new(),
308            last_event_id: String::new(),
309            retry: None,
310            bom_prefix: 0,
311            bom_done: false,
312            since_dispatch: 0,
313            offset: 0,
314            limit,
315            copied: 0,
316            refused: false,
317            recording: false,
318        }
319    }
320
321    /// Feed one transport chunk; yields every event it completed, then
322    /// [`TooLarge`] once an event holds more than the limit, after which it
323    /// yields nothing. The chunk goes in by pieces no larger than the room
324    /// left, so the framer never holds more than one byte past the limit.
325    /// An event holds its unfinished line, data, type and its copies of the
326    /// last id; comments and heartbeats hold nothing.
327    pub fn push(&mut self, chunk: &[u8]) -> std::vec::Drain<'_, Result<SseEvent, TooLarge>> {
328        let mut rest = chunk;
329        self.copied = 0;
330        while !self.refused && !rest.is_empty() {
331            let room = self.limit.saturating_sub(self.held()).saturating_add(1);
332            let (piece, after) = rest.split_at(room.min(rest.len()));
333            self.push_inner(piece, None);
334            rest = after;
335            if self.held() > self.limit {
336                self.refuse(self.held());
337            }
338        }
339        self.ready.drain(..)
340    }
341
342    /// The bytes of the current event it holds: its unfinished line, its
343    /// data, type and last id, and the ids copied into the events of this
344    /// push.
345    fn held(&self) -> usize {
346        self.buffer
347            .len()
348            .saturating_add(self.data.len())
349            .saturating_add(self.event_type.len())
350            .saturating_add(self.last_event_id.len())
351            .saturating_add(self.copied)
352    }
353
354    /// Refuses the current event, keeping the events before it.
355    fn refuse(&mut self, asked: usize) {
356        self.refused = true;
357        self.buffer = Vec::new();
358        self.data = String::new();
359        self.ready.push(Err(TooLarge {
360            limit: self.limit,
361            asked,
362        }));
363    }
364
365    /// Feed a chunk and retain parsed events with source ranges for recording.
366    pub fn push_recording(&mut self, chunk: &[u8]) -> SseBatch {
367        self.recording = true;
368        let mut batch = SseBatch::default();
369        self.push_inner(chunk, Some(&mut batch));
370        batch
371    }
372
373    /// Retain an incomplete final event for a recording, without changing
374    /// the normal SSE rule that EOF does not dispatch it.
375    pub fn finish_recording(&mut self) -> SseBatch {
376        self.recording = true;
377        let mut batch = SseBatch::default();
378        if self.bom_prefix > 0 {
379            self.buffer
380                .extend_from_slice(BOM.get(..self.bom_prefix).unwrap_or_default());
381            self.bom_prefix = 0;
382            self.bom_done = true;
383        }
384        if !self.buffer.is_empty() {
385            let buffer = std::mem::take(&mut self.buffer);
386            let start = self.offset;
387            self.offset = self.offset.saturating_add(buffer.len());
388            let (field, value) = self.line(&String::from_utf8_lossy(&buffer));
389            batch.lines.push(SseLine {
390                span: start..self.offset,
391                content: start..self.offset,
392                value: value.map(|at| start.saturating_add(at)..self.offset),
393                field,
394            });
395        }
396        self.dispatch();
397        if let Some(Ok(event)) = self.ready.pop() {
398            batch.events.push(RecordedSseEvent {
399                event,
400                end: self.offset,
401            });
402        }
403        batch
404    }
405
406    fn push_inner(&mut self, chunk: &[u8], mut recording: Option<&mut SseBatch>) {
407        let mut chunk = self.strip_bom(chunk);
408        if self.after_cr
409            && let Some((&first, rest)) = chunk.split_first()
410        {
411            self.after_cr = false;
412            if first == b'\n' {
413                chunk = rest;
414                self.offset = self.offset.saturating_add(1);
415                // Pending bytes are counted as if the CRLF had arrived whole.
416                if self.since_dispatch > 0 {
417                    self.since_dispatch = self.since_dispatch.saturating_add(1);
418                }
419            }
420        }
421        self.buffer.extend_from_slice(chunk);
422        // Taken out, so its lines are read in place while `line` changes the rest.
423        let buffer = std::mem::take(&mut self.buffer);
424        let mut rest = buffer.as_slice();
425        while let Some((line, next)) = terminated_line(rest, self.scanned) {
426            self.scanned = 0;
427            let start = self.offset;
428            self.offset = self.offset.saturating_add(rest.len() - next.len());
429            self.since_dispatch = self.since_dispatch.saturating_add(rest.len() - next.len());
430            self.after_cr = next.is_empty() && rest.last() == Some(&b'\r');
431            // A byte that is not UTF-8 becomes a three-byte replacement:
432            // refuse before a line grows past the limit that way.
433            let decoded = self.held().saturating_add(decoded_len(line));
434            if recording.is_none() && decoded > self.limit {
435                self.refuse(decoded);
436                return;
437            }
438            let (field, value) = self.line(&String::from_utf8_lossy(line));
439            if self.refused {
440                return;
441            }
442            // Past the limit, read no further line, so `push` refuses.
443            if recording.is_none() && self.held() > self.limit {
444                rest = next;
445                break;
446            }
447            if let Some(batch) = recording.as_deref_mut() {
448                batch.lines.push(SseLine {
449                    span: start..self.offset,
450                    content: start..start.saturating_add(line.len()),
451                    value: value
452                        .map(|at| start.saturating_add(at)..start.saturating_add(line.len())),
453                    field,
454                });
455                if let Some(Ok(event)) = self.ready.pop() {
456                    batch.events.push(RecordedSseEvent {
457                        event,
458                        end: self.offset,
459                    });
460                }
461            }
462            rest = next;
463        }
464        self.scanned = rest.len();
465        let consumed = buffer.len() - rest.len();
466        self.buffer = buffer;
467        self.buffer.drain(..consumed);
468    }
469
470    /// Saturating count of buffered bytes and complete lines since the last
471    /// blank line. Excludes any undecided leading BOM prefix; exposes no frame data.
472    pub fn pending(&self) -> usize {
473        self.since_dispatch.saturating_add(self.buffer.len())
474    }
475
476    /// Consume the stream's optional leading BOM, which may itself be split
477    /// across chunks. Returns the chunk with any BOM bytes removed.
478    fn strip_bom<'a>(&mut self, chunk: &'a [u8]) -> &'a [u8] {
479        if self.bom_done {
480            return chunk;
481        }
482        let mut rest = chunk;
483        while let Some(&byte) = rest.first() {
484            if BOM.get(self.bom_prefix) != Some(&byte) {
485                // Not a BOM after all: replay the bytes that matched so far.
486                let matched = std::mem::take(&mut self.bom_prefix);
487                self.bom_done = true;
488                self.buffer
489                    .extend_from_slice(BOM.get(..matched).unwrap_or_default());
490                return rest;
491            }
492            self.bom_prefix += 1;
493            rest = rest.get(1..).unwrap_or_default();
494            if self.bom_prefix == BOM.len() {
495                self.offset = self.offset.saturating_add(BOM.len());
496                self.bom_prefix = 0;
497                self.bom_done = true;
498                return rest;
499            }
500        }
501        rest
502    }
503
504    /// Process one complete line of the grammar.
505    fn line(&mut self, line: &str) -> (SseField, Option<usize>) {
506        if line.is_empty() {
507            self.dispatch();
508            return (SseField::Blank, None);
509        }
510        // A line starting with a colon is a comment.
511        if line.starts_with(':') {
512            return (SseField::Other, None);
513        }
514        let (field, value, at) = match line.split_once(':') {
515            Some((field, value)) => {
516                let trimmed = value.strip_prefix(' ').unwrap_or(value);
517                (
518                    field,
519                    trimmed,
520                    Some(field.len() + 1 + value.len() - trimmed.len()),
521                )
522            }
523            None => (line, "", None),
524        };
525        let field = match field {
526            "event" => {
527                self.event_type.clear();
528                self.event_type.push_str(value);
529                SseField::Event
530            }
531            "data" => {
532                self.data.push_str(value);
533                self.data.push('\n');
534                SseField::Data
535            }
536            // A NUL in an id is ignored outright, per the grammar.
537            "id" if !value.contains('\0') => {
538                self.last_event_id.clear();
539                self.last_event_id.push_str(value);
540                SseField::Id
541            }
542            "retry" if !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()) => {
543                // Digits only is a valid field, so a value past `u64` saturates.
544                self.retry = Some(value.parse().unwrap_or(u64::MAX));
545                SseField::Other
546            }
547            _ => SseField::Other,
548        };
549        (field, at)
550    }
551
552    /// A blank line ends an event. An event with no data is not dispatched:
553    /// it only resets the event type, as the grammar says.
554    fn dispatch(&mut self) {
555        self.since_dispatch = 0;
556        if self.data.is_empty() {
557            self.event_type.clear();
558            return;
559        }
560        self.data.pop();
561        // The event copies the last id: refuse before a copy passes the
562        // limit.
563        let asked = self.held().saturating_add(self.last_event_id.len());
564        if !self.recording && asked > self.limit {
565            return self.refuse(asked);
566        }
567        let event = if self.event_type.is_empty() {
568            Cow::Borrowed("message")
569        } else {
570            Cow::Owned(std::mem::take(&mut self.event_type))
571        };
572        self.copied = self.copied.saturating_add(self.last_event_id.len());
573        self.ready.push(Ok(SseEvent {
574            event,
575            data: std::mem::take(&mut self.data),
576            id: self.last_event_id.clone(),
577            retry: self.retry.take(),
578        }));
579        self.event_type.clear();
580    }
581}
582
583/// Splits nonempty newline-delimited payloads without validating JSON.
584/// A nonempty unterminated last line is available through [`Self::finish`].
585/// A line counts its bytes before its line end, LF or CRLF.
586#[derive(Debug)]
587pub struct NdjsonFramer {
588    buffer: Vec<u8>,
589    /// How many bytes at the start of `buffer` hold no line end.
590    scanned: usize,
591    ready: Vec<Result<Vec<u8>, TooLarge>>,
592    /// The most bytes of a line it holds.
593    limit: usize,
594    /// Whether it refused the line it reads, whose rest it skips.
595    skipping: bool,
596}
597
598impl Default for NdjsonFramer {
599    fn default() -> Self {
600        Self::with_limit(FRAME_LIMIT)
601    }
602}
603
604impl NdjsonFramer {
605    /// A framer at the start of a stream, refusing a line past
606    /// [`FRAME_LIMIT`].
607    pub fn new() -> Self {
608        Self::default()
609    }
610
611    /// A framer at the start of a stream, refusing a line of more than
612    /// `limit` bytes.
613    pub fn with_limit(limit: usize) -> Self {
614        Self {
615            buffer: Vec::new(),
616            scanned: 0,
617            ready: Vec::new(),
618            limit,
619            skipping: false,
620        }
621    }
622
623    /// Feed one transport chunk; yields every line it completed, and
624    /// [`TooLarge`] in the place of a line longer than the limit, whose
625    /// rest up to its LF it skips. Blank lines are not frames. The chunk
626    /// goes in by pieces no larger than the room left, so the framer never
627    /// holds more than one byte past the limit.
628    pub fn push(&mut self, chunk: &[u8]) -> std::vec::Drain<'_, Result<Vec<u8>, TooLarge>> {
629        let mut rest = chunk;
630        while !rest.is_empty() {
631            if self.skipping {
632                let Some(end) = rest.iter().position(|byte| *byte == b'\n') else {
633                    break;
634                };
635                self.skipping = false;
636                rest = rest.get(end + 1..).unwrap_or_default();
637                continue;
638            }
639            let room = self.limit.saturating_sub(self.held()).saturating_add(1);
640            let (piece, after) = rest.split_at(room.min(rest.len()));
641            self.lines(piece);
642            rest = after;
643            if self.held() > self.limit {
644                let asked = self.held();
645                self.buffer = Vec::new();
646                self.scanned = 0;
647                self.skipping = true;
648                self.ready.push(Err(TooLarge {
649                    limit: self.limit,
650                    asked,
651                }));
652            }
653        }
654        self.ready.drain(..)
655    }
656
657    /// The bytes of the line it holds; a CR at its end may be half of its
658    /// line end, so it does not count yet.
659    fn held(&self) -> usize {
660        let cr = self.buffer.last() == Some(&b'\r');
661        self.buffer.len() - usize::from(cr)
662    }
663
664    /// Takes the lines `chunk` completes, keeping its unterminated end.
665    fn lines(&mut self, chunk: &[u8]) {
666        self.buffer.extend_from_slice(chunk);
667        let mut rest = self.buffer.as_slice();
668        while let Some((line, end)) = split_line(rest, self.scanned, |byte| *byte == b'\n') {
669            self.scanned = 0;
670            let line = line.strip_suffix(b"\r").unwrap_or(line);
671            // A CR that was not the line's end counts after all.
672            if line.len() > self.limit {
673                self.ready.push(Err(TooLarge {
674                    limit: self.limit,
675                    asked: line.len(),
676                }));
677            } else if !line.is_empty() {
678                self.ready.push(Ok(line.to_vec()));
679            }
680            rest = end.get(1..).unwrap_or_default();
681        }
682        self.scanned = rest.len();
683        let consumed = self.buffer.len() - rest.len();
684        self.buffer.drain(..consumed);
685    }
686
687    /// The unterminated last line, if the stream ended with a nonempty one,
688    /// none after a refused one. A CR ending it is part of the line end, as
689    /// in [`Self::push`].
690    pub fn finish(&mut self) -> Option<Vec<u8>> {
691        let mut line = std::mem::take(&mut self.buffer);
692        if line.last() == Some(&b'\r') {
693            line.pop();
694        }
695        (!line.is_empty()).then_some(line)
696    }
697
698    /// Bytes of an unterminated trailing line, for truncation diagnostics.
699    pub fn pending(&self) -> usize {
700        self.buffer.len()
701    }
702}
703
704/// The bytes `line` takes once decoded, each run that is not UTF-8 as one
705/// replacement character.
706fn decoded_len(line: &[u8]) -> usize {
707    let replacement = char::REPLACEMENT_CHARACTER.len_utf8();
708    line.utf8_chunks().fold(0, |len, chunk| {
709        let invalid = if chunk.invalid().is_empty() {
710            0
711        } else {
712            replacement
713        };
714        len.saturating_add(chunk.valid().len())
715            .saturating_add(invalid)
716    })
717}
718
719/// The first complete line in `buffer`, without its line end, and the
720/// bytes after that line end; the first `scanned` bytes hold no line end.
721///
722/// A CR ends a line at once, as the grammar says; an LF right after it in
723/// `buffer` belongs to the same line end.
724fn terminated_line(buffer: &[u8], scanned: usize) -> Option<(&[u8], &[u8])> {
725    let (line, end) = split_line(buffer, scanned, |byte| *byte == b'\n' || *byte == b'\r')?;
726    let rest = end.strip_prefix(b"\r\n").or_else(|| end.get(1..));
727    Some((line, rest.unwrap_or_default()))
728}
729
730/// `buffer` split before its first byte that `ends` a line, if it has one;
731/// the first `scanned` bytes hold none.
732fn split_line(buffer: &[u8], scanned: usize, ends: impl Fn(&u8) -> bool) -> Option<(&[u8], &[u8])> {
733    buffer.split_at_checked(scanned + buffer.iter().skip(scanned).position(ends)?)
734}
735
736#[cfg(test)]
737mod tests;