Skip to main content

nu_protocol/pipeline/
byte_stream.rs

1//! Module managing the streaming of raw bytes between pipeline elements
2//!
3//! This module also handles conversions the [`ShellError`] <-> [`io::Error`](std::io::Error),
4//! so remember the usage of [`ShellErrorBridge`] where applicable.
5#[cfg(feature = "os")]
6use crate::process::{ChildPipe, ChildProcess};
7use crate::{
8    IntRange, PipelineData, ShellError, Signals, Span, Type, Value,
9    shell_error::{bridge::ShellErrorBridge, io::IoError},
10};
11use nu_utils::SplitRead as SplitReadInner;
12use serde::{Deserialize, Serialize};
13use std::ops::Bound;
14#[cfg(unix)]
15use std::os::fd::OwnedFd;
16#[cfg(windows)]
17use std::os::windows::io::OwnedHandle;
18use std::{
19    fmt::Debug,
20    fs::File,
21    io::{self, BufRead, BufReader, Cursor, ErrorKind, Read, Write},
22    process::Stdio,
23};
24
25/// The source of bytes for a [`ByteStream`].
26///
27/// Currently, there are only three possibilities:
28/// 1. `Read` (any `dyn` type that implements [`Read`])
29/// 2. [`File`]
30/// 3. [`ChildProcess`]
31pub enum ByteStreamSource {
32    Read(Box<dyn Read + Send + 'static>),
33    File(File),
34    #[cfg(feature = "os")]
35    Child(Box<ChildProcess>),
36}
37
38impl ByteStreamSource {
39    fn reader(self) -> Option<SourceReader> {
40        match self {
41            ByteStreamSource::Read(read) => Some(SourceReader::Read(read)),
42            ByteStreamSource::File(file) => Some(SourceReader::File(file)),
43            #[cfg(feature = "os")]
44            ByteStreamSource::Child(mut child) => child.stdout.take().map(|stdout| match stdout {
45                ChildPipe::Pipe(pipe) => SourceReader::File(convert_file(pipe)),
46                ChildPipe::Tee(tee) => SourceReader::Read(tee),
47            }),
48        }
49    }
50
51    /// Source is a `Child` or `File`, rather than `Read`. Currently affects trimming
52    #[cfg(feature = "os")]
53    pub fn is_external(&self) -> bool {
54        matches!(self, ByteStreamSource::Child(..))
55    }
56
57    #[cfg(not(feature = "os"))]
58    pub fn is_external(&self) -> bool {
59        // without os support we never have externals
60        false
61    }
62}
63
64impl Debug for ByteStreamSource {
65    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66        match self {
67            ByteStreamSource::Read(_) => f.debug_tuple("Read").field(&"..").finish(),
68            ByteStreamSource::File(file) => f.debug_tuple("File").field(file).finish(),
69            #[cfg(feature = "os")]
70            ByteStreamSource::Child(child) => f.debug_tuple("Child").field(child).finish(),
71        }
72    }
73}
74
75enum SourceReader {
76    Read(Box<dyn Read + Send + 'static>),
77    File(File),
78}
79
80impl Read for SourceReader {
81    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
82        match self {
83            SourceReader::Read(reader) => reader.read(buf),
84            SourceReader::File(file) => file.read(buf),
85        }
86    }
87}
88
89impl Debug for SourceReader {
90    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
91        match self {
92            SourceReader::Read(_) => f.debug_tuple("Read").field(&"..").finish(),
93            SourceReader::File(file) => f.debug_tuple("File").field(file).finish(),
94        }
95    }
96}
97
98/// Optional type color for [`ByteStream`], which determines type compatibility.
99#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
100pub enum ByteStreamType {
101    /// Compatible with [`Type::Binary`], and should only be converted to binary, even when the
102    /// desired type is unknown.
103    Binary,
104    /// Compatible with [`Type::String`], and should only be converted to string, even when the
105    /// desired type is unknown.
106    ///
107    /// This does not guarantee valid UTF-8 data, but it is conventionally so. Converting to
108    /// `String` still requires validation of the data.
109    String,
110    /// Unknown whether the stream should contain binary or string data. This usually is the result
111    /// of an external stream, e.g. an external command or file.
112    #[default]
113    Unknown,
114}
115
116impl ByteStreamType {
117    /// Returns the string that describes the byte stream type - i.e., the same as what `describe`
118    /// produces. This can be used in type mismatch error messages.
119    pub fn describe(self) -> &'static str {
120        match self {
121            ByteStreamType::Binary => "binary (stream)",
122            ByteStreamType::String => "string (stream)",
123            ByteStreamType::Unknown => "byte stream",
124        }
125    }
126
127    /// Returns true if the type is `Binary` or `Unknown`
128    pub fn is_binary_coercible(self) -> bool {
129        matches!(self, ByteStreamType::Binary | ByteStreamType::Unknown)
130    }
131
132    /// Returns true if the type is `String` or `Unknown`
133    pub fn is_string_coercible(self) -> bool {
134        matches!(self, ByteStreamType::String | ByteStreamType::Unknown)
135    }
136}
137
138impl From<ByteStreamType> for Type {
139    fn from(value: ByteStreamType) -> Self {
140        match value {
141            ByteStreamType::Binary => Type::Binary,
142            ByteStreamType::String => Type::String,
143            ByteStreamType::Unknown => Type::Any,
144        }
145    }
146}
147
148/// A potentially infinite, interruptible stream of bytes.
149///
150/// To create a [`ByteStream`], you can use any of the following methods:
151/// - [`read`](ByteStream::read): takes any type that implements [`Read`].
152/// - [`file`](ByteStream::file): takes a [`File`].
153/// - [`from_iter`](ByteStream::from_iter): takes an [`Iterator`] whose items implement `AsRef<[u8]>`.
154/// - [`from_result_iter`](ByteStream::from_result_iter): same as [`from_iter`](ByteStream::from_iter),
155///   but each item is a `Result<T, ShellError>`.
156/// - [`from_fn`](ByteStream::from_fn): uses a generator function to fill a buffer whenever it is
157///   empty. This has high performance because it doesn't need to allocate for each chunk of data,
158///   and can just reuse the same buffer.
159///
160/// Byte streams have a [type](.type_()) which is used to preserve type compatibility when they
161/// are the result of an internal command. It is important that this be set to the correct value.
162/// [`Unknown`](ByteStreamType::Unknown) is used only for external sources where the type can not
163/// be inherently determined, and having it automatically act as a string or binary depending on
164/// whether it parses as UTF-8 or not is desirable.
165///
166/// The data of a [`ByteStream`] can be accessed using one of the following methods:
167/// - [`reader`](ByteStream::reader): returns a [`Read`]-able type to get the raw bytes in the stream.
168/// - [`lines`](ByteStream::lines): splits the bytes on lines and returns an [`Iterator`]
169///   where each item is a `Result<String, ShellError>`.
170/// - [`chunks`](ByteStream::chunks): returns an [`Iterator`] of [`Value`]s where each value is
171///   either a string or binary.
172///   Try not to use this method if possible. Rather, please use [`reader`](ByteStream::reader)
173///   (or [`lines`](ByteStream::lines) if it matches the situation).
174///
175/// Additionally, there are few methods to collect a [`ByteStream`] into memory:
176/// - [`into_bytes`](ByteStream::into_bytes): collects all bytes into a [`Vec<u8>`].
177/// - [`into_string`](ByteStream::into_string): collects all bytes into a [`String`], erroring if utf-8 decoding failed.
178/// - [`into_value`](ByteStream::into_value): collects all bytes into a value typed appropriately
179///   for the [type](.type_()) of this stream. If the type is [`Unknown`](ByteStreamType::Unknown),
180///   it will produce a string value if the data is valid UTF-8, or a binary value otherwise.
181///
182/// There are also a few other methods to consume all the data of a [`ByteStream`]:
183/// - [`drain`](ByteStream::drain): consumes all bytes and outputs nothing.
184/// - [`write_to`](ByteStream::write_to): writes all bytes to the given [`Write`] destination.
185/// - [`print`](ByteStream::print): a convenience wrapper around [`write_to`](ByteStream::write_to).
186///   It prints all bytes to stdout or stderr.
187///
188/// Internally, [`ByteStream`]s currently come in three flavors according to [`ByteStreamSource`].
189/// See its documentation for more information.
190#[derive(Debug)]
191pub struct ByteStream {
192    stream: ByteStreamSource,
193    span: Span,
194    signals: Signals,
195    type_: ByteStreamType,
196    known_size: Option<u64>,
197    caller_spans: Vec<Span>,
198}
199
200impl ByteStream {
201    /// Create a new [`ByteStream`] from a [`ByteStreamSource`].
202    pub fn new(
203        stream: ByteStreamSource,
204        span: Span,
205        signals: Signals,
206        type_: ByteStreamType,
207    ) -> Self {
208        Self {
209            stream,
210            span,
211            signals,
212            type_,
213            known_size: None,
214            caller_spans: vec![],
215        }
216    }
217
218    /// Push a caller [`Span`] to the bytestream, it's useful to construct a backtrace.
219    pub fn push_caller_span(&mut self, span: Span) {
220        if span != self.span {
221            self.caller_spans.push(span)
222        }
223    }
224
225    /// Get all caller [`Span`], it's useful to construct a backtrace.
226    pub fn get_caller_spans(&self) -> &Vec<Span> {
227        &self.caller_spans
228    }
229
230    /// Create a [`ByteStream`] from an arbitrary reader. The type must be provided.
231    pub fn read(
232        reader: impl Read + Send + 'static,
233        span: Span,
234        signals: Signals,
235        type_: ByteStreamType,
236    ) -> Self {
237        Self::new(
238            ByteStreamSource::Read(Box::new(reader)),
239            span,
240            signals,
241            type_,
242        )
243    }
244
245    pub fn skip(self, span: Span, n: u64) -> Result<Self, ShellError> {
246        let known_size = self.known_size.map(|len| len.saturating_sub(n));
247        if let Some(mut reader) = self.reader() {
248            // Copy the number of skipped bytes into the sink before proceeding
249            io::copy(&mut (&mut reader).take(n), &mut io::sink())
250                .map_err(|err| IoError::new(err, span, None))?;
251            Ok(
252                ByteStream::read(reader, span, Signals::empty(), ByteStreamType::Binary)
253                    .with_known_size(known_size),
254            )
255        } else {
256            Err(ShellError::TypeMismatch {
257                err_message: "expected readable stream".into(),
258                span,
259            })
260        }
261    }
262
263    pub fn take(self, span: Span, n: u64) -> Result<Self, ShellError> {
264        let known_size = self.known_size.map(|s| s.min(n));
265        if let Some(reader) = self.reader() {
266            Ok(ByteStream::read(
267                reader.take(n),
268                span,
269                Signals::empty(),
270                ByteStreamType::Binary,
271            )
272            .with_known_size(known_size))
273        } else {
274            Err(ShellError::TypeMismatch {
275                err_message: "expected readable stream".into(),
276                span,
277            })
278        }
279    }
280
281    pub fn slice(
282        self,
283        val_span: Span,
284        call_span: Span,
285        range: IntRange,
286    ) -> Result<Self, ShellError> {
287        if let Some(len) = self.known_size {
288            let start = range.absolute_start(len);
289            let stream = self.skip(val_span, start);
290
291            match range.absolute_end(len) {
292                Bound::Unbounded => stream,
293                Bound::Included(end) | Bound::Excluded(end) if end < start => {
294                    stream.and_then(|s| s.take(val_span, 0))
295                }
296                Bound::Included(end) => {
297                    let distance = end - start + 1;
298                    stream.and_then(|s| s.take(val_span, distance.min(len)))
299                }
300                Bound::Excluded(end) => {
301                    let distance = end - start;
302                    stream.and_then(|s| s.take(val_span, distance.min(len)))
303                }
304            }
305        } else if range.is_relative() {
306            Err(ShellError::RelativeRangeOnInfiniteStream { span: call_span })
307        } else {
308            let start = range.start() as u64;
309            let stream = self.skip(val_span, start);
310
311            match range.distance() {
312                Bound::Unbounded => stream,
313                Bound::Included(distance) => stream.and_then(|s| s.take(val_span, distance + 1)),
314                Bound::Excluded(distance) => stream.and_then(|s| s.take(val_span, distance)),
315            }
316        }
317    }
318
319    /// Create a [`ByteStream`] from a string. The type of the stream is always `String`.
320    pub fn read_string(string: String, span: Span, signals: Signals) -> Self {
321        let len = string.len();
322        ByteStream::read(
323            Cursor::new(string.into_bytes()),
324            span,
325            signals,
326            ByteStreamType::String,
327        )
328        .with_known_size(Some(len as u64))
329    }
330
331    /// Create a [`ByteStream`] from binary data. The type of the stream is always `Binary`.
332    pub fn read_binary<T>(bytes: T, span: Span, signals: Signals) -> Self
333    where
334        T: AsRef<[u8]> + Send + 'static,
335    {
336        let len = bytes.as_ref().len();
337        ByteStream::read(Cursor::new(bytes), span, signals, ByteStreamType::Binary)
338            .with_known_size(Some(len as u64))
339    }
340
341    /// Create a [`ByteStream`] from a file.
342    ///
343    /// The type is implicitly `Unknown`, as it's not typically known whether files will
344    /// return text or binary.
345    pub fn file(file: File, span: Span, signals: Signals) -> Self {
346        Self::new(
347            ByteStreamSource::File(file),
348            span,
349            signals,
350            ByteStreamType::Unknown,
351        )
352    }
353
354    /// Create a [`ByteStream`] from a child process's stdout and stderr.
355    ///
356    /// The type is implicitly `Unknown`, as it's not typically known whether child processes will
357    /// return text or binary.
358    #[cfg(feature = "os")]
359    pub fn child(child: ChildProcess, span: Span) -> Self {
360        Self::new(
361            ByteStreamSource::Child(Box::new(child)),
362            span,
363            Signals::empty(),
364            ByteStreamType::Unknown,
365        )
366    }
367
368    /// Create a [`ByteStream`] that reads from stdin.
369    ///
370    /// The type is implicitly `Unknown`, as it's not typically known whether stdin is text or
371    /// binary.
372    #[cfg(feature = "os")]
373    pub fn stdin(span: Span) -> Result<Self, ShellError> {
374        let stdin = os_pipe::dup_stdin().map_err(|err| IoError::new(err, span, None))?;
375        let source = ByteStreamSource::File(convert_file(stdin));
376        Ok(Self::new(
377            source,
378            span,
379            Signals::empty(),
380            ByteStreamType::Unknown,
381        ))
382    }
383
384    #[cfg(not(feature = "os"))]
385    pub fn stdin(span: Span) -> Result<Self, ShellError> {
386        Err(ShellError::DisabledOsSupport {
387            msg: "Stdin is not supported".to_string(),
388            span,
389        })
390    }
391
392    /// Create a [`ByteStream`] from a generator function that writes data to the given buffer
393    /// when called, and returns `Ok(false)` on end of stream.
394    pub fn from_fn(
395        span: Span,
396        signals: Signals,
397        type_: ByteStreamType,
398        generator: impl FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
399    ) -> Self {
400        Self::read(
401            ReadGenerator {
402                buffer: Cursor::new(Vec::new()),
403                generator,
404            },
405            span,
406            signals,
407            type_,
408        )
409    }
410
411    pub fn with_type(mut self, type_: ByteStreamType) -> Self {
412        self.type_ = type_;
413        self
414    }
415
416    /// Create a new [`ByteStream`] from an [`Iterator`] of bytes slices.
417    ///
418    /// The returned [`ByteStream`] will have a [`ByteStreamSource`] of `Read`.
419    pub fn from_iter<I>(iter: I, span: Span, signals: Signals, type_: ByteStreamType) -> Self
420    where
421        I: IntoIterator,
422        I::IntoIter: Send + 'static,
423        I::Item: AsRef<[u8]> + Default + Send + 'static,
424    {
425        let iter = iter.into_iter();
426        let cursor = Some(Cursor::new(I::Item::default()));
427        Self::read(ReadIterator { iter, cursor }, span, signals, type_)
428    }
429
430    /// Create a new [`ByteStream`] from an [`Iterator`] of [`Result`] bytes slices.
431    ///
432    /// The returned [`ByteStream`] will have a [`ByteStreamSource`] of `Read`.
433    pub fn from_result_iter<I, T>(
434        iter: I,
435        span: Span,
436        signals: Signals,
437        type_: ByteStreamType,
438    ) -> Self
439    where
440        I: IntoIterator<Item = Result<T, ShellError>>,
441        I::IntoIter: Send + 'static,
442        T: AsRef<[u8]> + Default + Send + 'static,
443    {
444        let iter = iter.into_iter();
445        let cursor = Some(Cursor::new(T::default()));
446        Self::read(ReadResultIterator { iter, cursor }, span, signals, type_)
447    }
448
449    /// Set the known size, in number of bytes, of the [`ByteStream`].
450    pub fn with_known_size(mut self, size: Option<u64>) -> Self {
451        self.known_size = size;
452        self
453    }
454
455    /// Get a reference to the inner [`ByteStreamSource`] of the [`ByteStream`].
456    pub fn source(&self) -> &ByteStreamSource {
457        &self.stream
458    }
459
460    /// Get a mutable reference to the inner [`ByteStreamSource`] of the [`ByteStream`].
461    pub fn source_mut(&mut self) -> &mut ByteStreamSource {
462        &mut self.stream
463    }
464
465    /// Returns the [`Span`] associated with the [`ByteStream`].
466    pub fn span(&self) -> Span {
467        self.span
468    }
469
470    /// Changes the [`Span`] associated with the [`ByteStream`].
471    pub fn with_span(mut self, span: Span) -> Self {
472        self.span = span;
473        self
474    }
475
476    /// Returns the [`ByteStreamType`] associated with the [`ByteStream`].
477    /// Process interrupt signals associated with this stream.
478    pub fn signals(&self) -> &Signals {
479        &self.signals
480    }
481
482    pub fn type_(&self) -> ByteStreamType {
483        self.type_
484    }
485
486    /// Returns the known size, in number of bytes, of the [`ByteStream`].
487    pub fn known_size(&self) -> Option<u64> {
488        self.known_size
489    }
490
491    /// Convert the [`ByteStream`] into its [`Reader`] which allows one to [`Read`] the raw bytes of the stream.
492    ///
493    /// [`Reader`] is buffered and also implements [`BufRead`].
494    ///
495    /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout,
496    /// then the stream is considered empty and `None` will be returned.
497    pub fn reader(self) -> Option<Reader> {
498        let reader = self.stream.reader()?;
499        Some(Reader {
500            reader: BufReader::new(reader),
501            span: self.span,
502            signals: self.signals,
503        })
504    }
505
506    /// Convert the [`ByteStream`] into a [`Lines`] iterator where each element is a `Result<String, ShellError>`.
507    ///
508    /// There is no limit on how large each line will be. Ending new lines (`\n` or `\r\n`) are
509    /// stripped from each line. If a line fails to be decoded as utf-8, then it will become a [`ShellError`].
510    ///
511    /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout,
512    /// then the stream is considered empty and `None` will be returned.
513    pub fn lines(self) -> Option<Lines> {
514        let reader = self.stream.reader()?;
515        Some(Lines {
516            reader: BufReader::new(reader),
517            span: self.span,
518            signals: self.signals,
519            strict: false,
520        })
521    }
522
523    /// Convert the [`ByteStream`] into a [`SplitRead`] iterator where each element is a `Result<String, ShellError>`.
524    ///
525    /// Each call to [`next`](Iterator::next) reads the currently available data from the byte
526    /// stream source, until `delimiter` or the end of the stream is encountered.
527    ///
528    /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout,
529    /// then the stream is considered empty and `None` will be returned.
530    pub fn split(self, delimiter: Vec<u8>) -> Option<SplitRead> {
531        let reader = self.stream.reader()?;
532        Some(SplitRead::new(reader, delimiter, self.span, self.signals))
533    }
534
535    /// Convert the [`ByteStream`] into a [`Chunks`] iterator where each element is a `Result<Value, ShellError>`.
536    ///
537    /// Each call to [`next`](Iterator::next) reads the currently available data from the byte stream source,
538    /// up to a maximum size. The values are typed according to the [type](.type_()) of the
539    /// stream, and if that type is [`Unknown`](ByteStreamType::Unknown), string values will be
540    /// produced as long as the stream continues to parse as valid UTF-8, but binary values will
541    /// be produced instead of the stream fails to parse as UTF-8 instead at any point.
542    /// Any and all newlines are kept intact in each chunk.
543    ///
544    /// Where possible, prefer [`reader`](ByteStream::reader) or [`lines`](ByteStream::lines) over this method.
545    /// Those methods are more likely to be used in a semantically correct way
546    /// (and [`reader`](ByteStream::reader) is more efficient too).
547    ///
548    /// If the source of the [`ByteStream`] is [`ByteStreamSource::Child`] and the child has no stdout,
549    /// then the stream is considered empty and `None` will be returned.
550    pub fn chunks(self) -> Option<Chunks> {
551        let reader = self.stream.reader()?;
552        Some(Chunks::new(reader, self.span, self.signals, self.type_))
553    }
554
555    /// Convert the [`ByteStream`] into its inner [`ByteStreamSource`].
556    pub fn into_source(self) -> ByteStreamSource {
557        self.stream
558    }
559
560    /// Attempt to convert the [`ByteStream`] into a [`Stdio`].
561    ///
562    /// This will succeed if the [`ByteStreamSource`] of the [`ByteStream`] is either:
563    /// - [`File`](ByteStreamSource::File)
564    /// - [`Child`](ByteStreamSource::Child) and the child has a stdout that is `Some(ChildPipe::Pipe(..))`.
565    ///
566    /// All other cases return an `Err` with the original [`ByteStream`] in it.
567    pub fn into_stdio(mut self) -> Result<Stdio, Self> {
568        match self.stream {
569            ByteStreamSource::Read(..) => Err(self),
570            ByteStreamSource::File(file) => Ok(file.into()),
571            #[cfg(feature = "os")]
572            ByteStreamSource::Child(child) => {
573                if let ChildProcess {
574                    stdout: Some(ChildPipe::Pipe(stdout)),
575                    stderr,
576                    ..
577                } = *child
578                {
579                    debug_assert!(stderr.is_none(), "stderr should not exist");
580                    Ok(stdout.into())
581                } else {
582                    self.stream = ByteStreamSource::Child(child);
583                    Err(self)
584                }
585            }
586        }
587    }
588
589    /// Attempt to convert the [`ByteStream`] into a [`ChildProcess`].
590    ///
591    /// This will only succeed if the [`ByteStreamSource`] of the [`ByteStream`] is [`Child`](ByteStreamSource::Child).
592    /// All other cases return an `Err` with the original [`ByteStream`] in it.
593    #[cfg(feature = "os")]
594    pub fn into_child(self) -> Result<ChildProcess, Self> {
595        if let ByteStreamSource::Child(child) = self.stream {
596            Ok(*child)
597        } else {
598            Err(self)
599        }
600    }
601
602    /// Collect all the bytes of the [`ByteStream`] into a [`Vec<u8>`].
603    ///
604    /// Any trailing new lines are kept in the returned [`Vec`].
605    pub fn into_bytes(self) -> Result<Vec<u8>, ShellError> {
606        let span = self.span;
607        let signals = self.signals;
608        let from_io_error = IoError::factory(span, None);
609        match self.stream {
610            ByteStreamSource::Read(mut read) => {
611                let mut buf = Vec::new();
612                let mut chunk = [0; DEFAULT_BUF_SIZE];
613                loop {
614                    signals.check(&span)?;
615                    match read.read(&mut chunk) {
616                        Ok(0) => break,
617                        Ok(n) => buf.extend_from_slice(&chunk[..n]),
618                        Err(e) if e.kind() == ErrorKind::Interrupted => continue,
619                        Err(e) => match ShellErrorBridge::try_from(e) {
620                            Ok(ShellErrorBridge(e)) => return Err(e),
621                            Err(e) => return Err(ShellError::Io(from_io_error(e))),
622                        },
623                    }
624                }
625                Ok(buf)
626            }
627            ByteStreamSource::File(mut file) => {
628                let mut buf = Vec::new();
629                let mut chunk = [0; DEFAULT_BUF_SIZE];
630                loop {
631                    signals.check(&span)?;
632                    match file.read(&mut chunk) {
633                        Ok(0) => break,
634                        Ok(n) => buf.extend_from_slice(&chunk[..n]),
635                        Err(e) if e.kind() == ErrorKind::Interrupted => continue,
636                        Err(e) => return Err(ShellError::Io(from_io_error(e))),
637                    }
638                }
639                Ok(buf)
640            }
641            #[cfg(feature = "os")]
642            ByteStreamSource::Child(child) => child.into_bytes(),
643        }
644    }
645
646    /// Collect the stream into a `String` in-memory. This can only succeed if the data contained is
647    /// valid UTF-8.
648    ///
649    /// The trailing new line (`\n` or `\r\n`), if any, is removed from the [`String`] prior to
650    /// being returned, if this is a stream coming from an external process or file.
651    ///
652    /// If the [type](.type_()) is specified as `Binary`, this operation always fails, even if the
653    /// data would have been valid UTF-8.
654    pub fn into_string(self) -> Result<String, ShellError> {
655        let span = self.span;
656        if self.type_.is_string_coercible() {
657            let trim = self.stream.is_external();
658            let bytes = self.into_bytes()?;
659            let mut string = String::from_utf8(bytes).map_err(|err| ShellError::NonUtf8Custom {
660                span,
661                msg: err.to_string(),
662            })?;
663            if trim {
664                trim_end_newline(&mut string);
665            }
666            Ok(string)
667        } else {
668            Err(ShellError::TypeMismatch {
669                err_message: "expected string, but got binary".into(),
670                span,
671            })
672        }
673    }
674
675    /// Collect all the bytes of the [`ByteStream`] into a [`Value`].
676    ///
677    /// If this is a `String` stream, the stream is decoded to UTF-8. If the stream came from an
678    /// external process or file, the trailing new line (`\n` or `\r\n`), if any, is removed from
679    /// the [`String`] prior to being returned.
680    ///
681    /// If this is a `Binary` stream, a [`Value::Binary`] is returned with any trailing new lines
682    /// preserved.
683    ///
684    /// If this is an `Unknown` stream, the behavior depends on whether the stream parses as valid
685    /// UTF-8 or not. If it does, this is uses the `String` behavior; if not, it uses the `Binary`
686    /// behavior.
687    pub fn into_value(self) -> Result<Value, ShellError> {
688        let span = self.span;
689        let trim = self.stream.is_external();
690        let value = match self.type_ {
691            // If the type is specified, then the stream should always become that type:
692            ByteStreamType::Binary => Value::binary(self.into_bytes()?, span),
693            // Fail on invalid UTF-8 rather than falling back to binary (unlike [`value_from_bytes`]).
694            ByteStreamType::String => Value::string(self.into_string()?, span),
695            // If the type is not specified, then it just depends on whether it parses or not:
696            ByteStreamType::Unknown => {
697                value_from_bytes(self.into_bytes()?, span, ByteStreamType::Unknown, trim)
698            }
699        };
700        Ok(value)
701    }
702
703    /// Consume and drop all bytes of the [`ByteStream`].
704    pub fn drain(self) -> Result<(), ShellError> {
705        match self.stream {
706            ByteStreamSource::Read(read) => {
707                copy_with_signals(read, io::sink(), self.span, &self.signals)?;
708                Ok(())
709            }
710            ByteStreamSource::File(_) => Ok(()),
711            #[cfg(feature = "os")]
712            ByteStreamSource::Child(child) => child.wait(),
713        }
714    }
715
716    /// Print all bytes of the [`ByteStream`] to stdout or stderr.
717    pub fn print(self, to_stderr: bool) -> Result<(), ShellError> {
718        if to_stderr {
719            self.write_to(&mut io::stderr())
720        } else {
721            self.write_to(&mut io::stdout())
722        }
723    }
724
725    /// Write all bytes of the [`ByteStream`] to `dest`.
726    pub fn write_to(self, dest: impl Write) -> Result<(), ShellError> {
727        let span = self.span;
728        let signals = &self.signals;
729        match self.stream {
730            ByteStreamSource::Read(read) => {
731                copy_with_signals(read, dest, span, signals)?;
732            }
733            ByteStreamSource::File(file) => {
734                copy_with_signals(file, dest, span, signals)?;
735            }
736            #[cfg(feature = "os")]
737            ByteStreamSource::Child(mut child) => {
738                // All `OutDest`s except `OutDest::PipeSeparate` will cause `stderr` to be `None`.
739                // Only `save`, `tee`, and `complete` set the stderr `OutDest` to `OutDest::PipeSeparate`,
740                // and those commands have proper simultaneous handling of stdout and stderr.
741                debug_assert!(child.stderr.is_none(), "stderr should not exist");
742
743                if let Some(stdout) = child.stdout.take() {
744                    match stdout {
745                        ChildPipe::Pipe(pipe) => {
746                            copy_with_signals(pipe, dest, span, signals)?;
747                        }
748                        ChildPipe::Tee(tee) => {
749                            copy_with_signals(tee, dest, span, signals)?;
750                        }
751                    }
752                }
753                child.wait()?;
754            }
755        }
756        Ok(())
757    }
758
759    /// The process of an external whose stderr has its own pipe, which happens when the next
760    /// command reads stdout and stderr apart (`complete`, `ignore`, `save`, `tee`).
761    #[cfg(feature = "os")]
762    pub fn child_with_separate_stderr(&mut self) -> Option<&mut ChildProcess> {
763        match &mut self.stream {
764            ByteStreamSource::Child(child) if child.stderr.is_some() => Some(child),
765            _ => None,
766        }
767    }
768
769    /// For an external whose stderr has its own pipe, a copy of the stream without the stderr,
770    /// reading the same stdout from memory (see [`ChildProcess::copy_stdout`]). The stream itself
771    /// still reads the same. `None` for any other stream.
772    #[cfg(feature = "os")]
773    pub fn copy_stdout(&mut self) -> Result<Option<Self>, ShellError> {
774        let (span, signals, type_) = (self.span, self.signals.clone(), self.type_);
775        let Some(child) = self.child_with_separate_stderr() else {
776            return Ok(None);
777        };
778        let copy = ByteStreamSource::Child(Box::new(child.copy_stdout()?));
779        Ok(Some(Self::new(copy, span, signals, type_)))
780    }
781
782    /// Write the stream to `dest` like [`write_to`](Self::write_to), unless it's an external
783    /// whose stderr has its own pipe, meaning the next command reads stdout and stderr apart.
784    /// Then only stdout is written, and the stream comes back with the process and its stderr.
785    #[cfg(feature = "os")]
786    pub fn write_stdout_to(mut self, dest: impl Write) -> Result<Option<Self>, ShellError> {
787        let ByteStreamSource::Child(child) = &mut self.stream else {
788            return self.write_to(dest).map(|()| None);
789        };
790        if child.stderr.is_none() {
791            return self.write_to(dest).map(|()| None);
792        }
793        if child.stdout.is_some() {
794            // Read both pipes first, so the child can't block on stderr while stdout is written.
795            child.buffer_pipes()?;
796            if let Some(stdout) = child.stdout.take() {
797                copy_with_signals(stdout, dest, self.span, &self.signals)?;
798            }
799        }
800        Ok(Some(self))
801    }
802}
803
804impl From<ByteStream> for PipelineData {
805    fn from(stream: ByteStream) -> Self {
806        Self::byte_stream(stream, None)
807    }
808}
809
810struct ReadIterator<I>
811where
812    I: Iterator,
813    I::Item: AsRef<[u8]>,
814{
815    iter: I,
816    cursor: Option<Cursor<I::Item>>,
817}
818
819impl<I> Read for ReadIterator<I>
820where
821    I: Iterator,
822    I::Item: AsRef<[u8]>,
823{
824    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
825        while let Some(cursor) = self.cursor.as_mut() {
826            let read = cursor.read(buf)?;
827            if read == 0 {
828                self.cursor = self.iter.next().map(Cursor::new);
829            } else {
830                return Ok(read);
831            }
832        }
833        Ok(0)
834    }
835}
836
837struct ReadResultIterator<I, T>
838where
839    I: Iterator<Item = Result<T, ShellError>>,
840    T: AsRef<[u8]>,
841{
842    iter: I,
843    cursor: Option<Cursor<T>>,
844}
845
846impl<I, T> Read for ReadResultIterator<I, T>
847where
848    I: Iterator<Item = Result<T, ShellError>>,
849    T: AsRef<[u8]>,
850{
851    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
852        while let Some(cursor) = self.cursor.as_mut() {
853            let read = cursor.read(buf)?;
854            if read == 0 {
855                self.cursor = self
856                    .iter
857                    .next()
858                    .transpose()
859                    .map_err(ShellErrorBridge)?
860                    .map(Cursor::new);
861            } else {
862                return Ok(read);
863            }
864        }
865        Ok(0)
866    }
867}
868
869pub struct Reader {
870    reader: BufReader<SourceReader>,
871    span: Span,
872    signals: Signals,
873}
874
875impl Reader {
876    pub fn span(&self) -> Span {
877        self.span
878    }
879}
880
881impl Read for Reader {
882    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
883        self.signals.check(&self.span).map_err(ShellErrorBridge)?;
884        self.reader.read(buf)
885    }
886}
887
888impl BufRead for Reader {
889    fn fill_buf(&mut self) -> io::Result<&[u8]> {
890        self.reader.fill_buf()
891    }
892
893    fn consume(&mut self, amt: usize) {
894        self.reader.consume(amt)
895    }
896}
897
898pub struct Lines {
899    reader: BufReader<SourceReader>,
900    span: Span,
901    signals: Signals,
902    /// Controls UTF-8 decoding behavior for each line.
903    ///
904    /// When `false` (the default), invalid UTF-8 bytes are replaced with the Unicode
905    /// replacement character (U+FFFD) using lossy conversion, so processing continues
906    /// uninterrupted even if the input is not valid UTF-8.
907    ///
908    /// When `true`, any line containing invalid UTF-8 bytes will immediately produce
909    /// a [`ShellError::NonUtf8`] error instead of a string value.
910    strict: bool,
911}
912
913impl Lines {
914    pub fn span(&self) -> Span {
915        self.span
916    }
917
918    /// Sets the UTF-8 decoding mode for this iterator.
919    ///
920    /// When `strict` is `true`, any line that contains invalid UTF-8 bytes will yield
921    /// a [`ShellError::NonUtf8`] error. When `false` (the default), invalid bytes are
922    /// silently replaced with the Unicode replacement character (U+FFFD `\u{FFFD}`).
923    pub fn strict(mut self, strict: bool) -> Self {
924        self.strict = strict;
925        self
926    }
927}
928
929impl Iterator for Lines {
930    type Item = Result<String, ShellError>;
931
932    fn next(&mut self) -> Option<Self::Item> {
933        if self.signals.interrupted() {
934            None
935        } else {
936            let mut buf = Vec::new();
937            match self.reader.read_until(b'\n', &mut buf) {
938                Ok(0) => None,
939                Ok(_) => {
940                    let mut string = if self.strict {
941                        match String::from_utf8(buf) {
942                            Ok(s) => s,
943                            Err(_) => return Some(Err(ShellError::NonUtf8 { span: self.span })),
944                        }
945                    } else {
946                        String::from_utf8_lossy(&buf).into_owned()
947                    };
948                    trim_end_newline(&mut string);
949                    Some(Ok(string))
950                }
951                Err(err) => Some(Err(IoError::new(err, self.span, None).into())),
952            }
953        }
954    }
955}
956
957pub struct SplitRead {
958    internal: SplitReadInner<BufReader<SourceReader>>,
959    span: Span,
960    signals: Signals,
961}
962
963impl SplitRead {
964    fn new(
965        reader: SourceReader,
966        delimiter: impl AsRef<[u8]>,
967        span: Span,
968        signals: Signals,
969    ) -> Self {
970        Self {
971            internal: SplitReadInner::new(BufReader::new(reader), delimiter),
972            span,
973            signals,
974        }
975    }
976
977    pub fn span(&self) -> Span {
978        self.span
979    }
980}
981
982impl Iterator for SplitRead {
983    type Item = Result<Vec<u8>, ShellError>;
984
985    fn next(&mut self) -> Option<Self::Item> {
986        if self.signals.interrupted() {
987            return None;
988        }
989        self.internal.next().map(|r| {
990            r.map_err(|err| {
991                ShellError::Io(IoError::new_internal(
992                    err,
993                    "Could not get next value for SplitRead",
994                ))
995            })
996        })
997    }
998}
999
1000/// Turn a readable stream into [`Value`]s.
1001///
1002/// The `Value` type depends on the type of the stream ([`ByteStreamType`]). If `Unknown`, the
1003/// stream will return strings as long as UTF-8 parsing succeeds, but will start returning binary
1004/// if it fails.
1005pub struct Chunks {
1006    reader: BufReader<SourceReader>,
1007    pos: u64,
1008    error: bool,
1009    span: Span,
1010    signals: Signals,
1011    type_: ByteStreamType,
1012}
1013
1014impl Chunks {
1015    fn new(reader: SourceReader, span: Span, signals: Signals, type_: ByteStreamType) -> Self {
1016        Self {
1017            reader: BufReader::new(reader),
1018            pos: 0,
1019            error: false,
1020            span,
1021            signals,
1022            type_,
1023        }
1024    }
1025
1026    pub fn span(&self) -> Span {
1027        self.span
1028    }
1029
1030    fn next_string(&mut self) -> Result<Option<String>, (Vec<u8>, ShellError)> {
1031        let from_io_error = |err: std::io::Error| match ShellErrorBridge::try_from(err) {
1032            Ok(err) => err.0,
1033            Err(err) => IoError::new(err, self.span, None).into(),
1034        };
1035
1036        // Get some data from the reader
1037        let buf = self
1038            .reader
1039            .fill_buf()
1040            .map_err(from_io_error)
1041            .map_err(|err| (vec![], err))?;
1042
1043        // If empty, this is EOF
1044        if buf.is_empty() {
1045            return Ok(None);
1046        }
1047
1048        let mut buf = buf.to_vec();
1049        let mut consumed = 0;
1050
1051        // If the buf length is under 4 bytes, it could be invalid, so try to get more
1052        if buf.len() < 4 {
1053            consumed += buf.len();
1054            self.reader.consume(buf.len());
1055            match self.reader.fill_buf() {
1056                Ok(more_bytes) => buf.extend_from_slice(more_bytes),
1057                Err(err) => return Err((buf, from_io_error(err))),
1058            }
1059        }
1060
1061        // Try to parse utf-8 and decide what to do
1062        match String::from_utf8(buf) {
1063            Ok(string) => {
1064                self.reader.consume(string.len() - consumed);
1065                self.pos += string.len() as u64;
1066                Ok(Some(string))
1067            }
1068            Err(err) if err.utf8_error().error_len().is_none() => {
1069                // There is some valid data at the beginning, and this is just incomplete, so just
1070                // consume that and return it
1071                let valid_up_to = err.utf8_error().valid_up_to();
1072                if valid_up_to > consumed {
1073                    self.reader.consume(valid_up_to - consumed);
1074                }
1075                let mut buf = err.into_bytes();
1076                buf.truncate(valid_up_to);
1077                buf.shrink_to_fit();
1078                let string = String::from_utf8(buf)
1079                    .expect("failed to parse utf-8 even after correcting error");
1080                self.pos += string.len() as u64;
1081                Ok(Some(string))
1082            }
1083            Err(err) => {
1084                // There is an error at the beginning and we have no hope of parsing further.
1085                let shell_error = ShellError::NonUtf8Custom {
1086                    msg: format!("invalid utf-8 sequence starting at index {}", self.pos),
1087                    span: self.span,
1088                };
1089                let buf = err.into_bytes();
1090                // We are consuming the entire buf though, because we're returning it in case it
1091                // will be cast to binary
1092                if buf.len() > consumed {
1093                    self.reader.consume(buf.len() - consumed);
1094                }
1095                self.pos += buf.len() as u64;
1096                Err((buf, shell_error))
1097            }
1098        }
1099    }
1100}
1101
1102impl Iterator for Chunks {
1103    type Item = Result<Value, ShellError>;
1104
1105    fn next(&mut self) -> Option<Self::Item> {
1106        if self.error || self.signals.interrupted() {
1107            None
1108        } else {
1109            match self.type_ {
1110                // Binary should always be binary
1111                ByteStreamType::Binary => {
1112                    let buf = match self.reader.fill_buf() {
1113                        Ok(buf) => buf,
1114                        Err(err) => {
1115                            self.error = true;
1116                            return Some(Err(ShellError::Io(IoError::new(err, self.span, None))));
1117                        }
1118                    };
1119                    if !buf.is_empty() {
1120                        let len = buf.len();
1121                        let value = Value::binary(buf, self.span);
1122                        self.reader.consume(len);
1123                        self.pos += len as u64;
1124                        Some(Ok(value))
1125                    } else {
1126                        None
1127                    }
1128                }
1129                // String produces an error if UTF-8 can't be parsed
1130                ByteStreamType::String => match self.next_string().transpose()? {
1131                    Ok(string) => Some(Ok(Value::string(string, self.span))),
1132                    Err((_, err)) => {
1133                        self.error = true;
1134                        Some(Err(err))
1135                    }
1136                },
1137                // For Unknown, we try to create strings, but we switch to binary mode if we
1138                // fail
1139                ByteStreamType::Unknown => {
1140                    match self.next_string().transpose()? {
1141                        Ok(string) => Some(Ok(Value::string(string, self.span))),
1142                        Err((buf, _)) if !buf.is_empty() => {
1143                            // Switch to binary mode
1144                            self.type_ = ByteStreamType::Binary;
1145                            Some(Ok(Value::binary(buf, self.span)))
1146                        }
1147                        Err((_, err)) => {
1148                            self.error = true;
1149                            Some(Err(err))
1150                        }
1151                    }
1152                }
1153            }
1154        }
1155    }
1156}
1157
1158fn trim_end_newline(string: &mut String) {
1159    if string.ends_with('\n') {
1160        string.pop();
1161        if string.ends_with('\r') {
1162            string.pop();
1163        }
1164    }
1165}
1166
1167/// Convert already-collected stream bytes into a [`Value`].
1168///
1169/// Shared by [`ByteStream::into_value`] (for [`ByteStreamType::Unknown`]) and interactive
1170/// last-result capture of a byte-stream prefix.
1171///
1172/// - [`ByteStreamType::Binary`] → binary (no decode)
1173/// - [`ByteStreamType::String`] / [`ByteStreamType::Unknown`] → UTF-8 string when valid, else binary
1174///
1175/// Incomplete multi-byte sequences at the end (typical when a prefix is truncated to a budget)
1176/// keep the valid UTF-8 prefix as a string rather than failing the whole buffer to binary.
1177///
1178/// When `trim_trailing_newline` is true, a single trailing `\n` or `\r\n` is stripped, matching
1179/// collected external/file values.
1180pub fn value_from_bytes(
1181    bytes: Vec<u8>,
1182    span: Span,
1183    type_: ByteStreamType,
1184    trim_trailing_newline: bool,
1185) -> Value {
1186    if matches!(type_, ByteStreamType::Binary) {
1187        return Value::binary(bytes, span);
1188    }
1189
1190    match String::from_utf8(bytes) {
1191        Ok(mut s) => {
1192            if trim_trailing_newline {
1193                trim_end_newline(&mut s);
1194            }
1195            Value::string(s, span)
1196        }
1197        Err(err) => {
1198            let valid_up_to = err.utf8_error().valid_up_to();
1199            // `error_len() == None` means unexpected EOF mid-sequence (truncation).
1200            let incomplete_at_end = err.utf8_error().error_len().is_none();
1201            let bytes = err.into_bytes();
1202            if incomplete_at_end && valid_up_to > 0 {
1203                // SAFETY: `valid_up_to` is the end of a valid UTF-8 prefix.
1204                let mut s = String::from_utf8(bytes[..valid_up_to].to_vec())
1205                    .expect("valid_up_to marks a valid UTF-8 prefix");
1206                if trim_trailing_newline {
1207                    trim_end_newline(&mut s);
1208                }
1209                Value::string(s, span)
1210            } else {
1211                Value::binary(bytes, span)
1212            }
1213        }
1214    }
1215}
1216
1217#[cfg(unix)]
1218pub(crate) fn convert_file<T: From<OwnedFd>>(file: impl Into<OwnedFd>) -> T {
1219    file.into().into()
1220}
1221
1222#[cfg(windows)]
1223pub(crate) fn convert_file<T: From<OwnedHandle>>(file: impl Into<OwnedHandle>) -> T {
1224    file.into().into()
1225}
1226
1227const DEFAULT_BUF_SIZE: usize = 8192;
1228
1229pub fn copy_with_signals(
1230    mut reader: impl Read,
1231    mut writer: impl Write,
1232    span: Span,
1233    signals: &Signals,
1234) -> Result<u64, ShellError> {
1235    let from_io_error = IoError::factory(span, None);
1236    if signals.is_empty() {
1237        match io::copy(&mut reader, &mut writer) {
1238            Ok(n) => {
1239                writer.flush().map_err(&from_io_error)?;
1240                Ok(n)
1241            }
1242            Err(err) => {
1243                let _ = writer.flush();
1244                match ShellErrorBridge::try_from(err) {
1245                    Ok(ShellErrorBridge(shell_error)) => Err(shell_error),
1246                    Err(err) => Err(from_io_error(err).into()),
1247                }
1248            }
1249        }
1250    } else {
1251        // #[cfg(any(target_os = "linux", target_os = "android"))]
1252        // {
1253        //     return crate::sys::kernel_copy::copy_spec(reader, writer);
1254        // }
1255        match generic_copy(&mut reader, &mut writer, span, signals) {
1256            Ok(len) => {
1257                writer.flush().map_err(&from_io_error)?;
1258                Ok(len)
1259            }
1260            Err(err) => {
1261                let _ = writer.flush();
1262                Err(err)
1263            }
1264        }
1265    }
1266}
1267
1268// Copied from [`std::io::copy`]
1269fn generic_copy(
1270    mut reader: impl Read,
1271    mut writer: impl Write,
1272    span: Span,
1273    signals: &Signals,
1274) -> Result<u64, ShellError> {
1275    let from_io_error = IoError::factory(span, None);
1276    let buf = &mut [0; DEFAULT_BUF_SIZE];
1277    let mut len = 0;
1278    loop {
1279        signals.check(&span)?;
1280        let n = match reader.read(buf) {
1281            Ok(0) => break,
1282            Ok(n) => n,
1283            Err(e) if e.kind() == ErrorKind::Interrupted => continue,
1284            Err(e) => match ShellErrorBridge::try_from(e) {
1285                Ok(ShellErrorBridge(e)) => return Err(e),
1286                Err(e) => return Err(from_io_error(e).into()),
1287            },
1288        };
1289        len += n;
1290        writer.write_all(&buf[..n]).map_err(&from_io_error)?;
1291    }
1292    Ok(len as u64)
1293}
1294
1295struct ReadGenerator<F>
1296where
1297    F: FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
1298{
1299    buffer: Cursor<Vec<u8>>,
1300    generator: F,
1301}
1302
1303impl<F> BufRead for ReadGenerator<F>
1304where
1305    F: FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
1306{
1307    fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
1308        // We have to loop, because it's important that we don't leave the buffer empty unless we're
1309        // truly at the end of the stream.
1310        while self.buffer.fill_buf()?.is_empty() {
1311            // Reset the cursor to the beginning and truncate
1312            self.buffer.set_position(0);
1313            self.buffer.get_mut().clear();
1314            // Ask the generator to generate data
1315            if !(self.generator)(self.buffer.get_mut()).map_err(ShellErrorBridge)? {
1316                // End of stream
1317                break;
1318            }
1319        }
1320        self.buffer.fill_buf()
1321    }
1322
1323    fn consume(&mut self, amt: usize) {
1324        self.buffer.consume(amt);
1325    }
1326}
1327
1328impl<F> Read for ReadGenerator<F>
1329where
1330    F: FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
1331{
1332    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1333        // Straightforward implementation on top of BufRead
1334        let slice = self.fill_buf()?;
1335        let len = buf.len().min(slice.len());
1336        buf[..len].copy_from_slice(&slice[..len]);
1337        self.consume(len);
1338        Ok(len)
1339    }
1340}
1341
1342#[cfg(test)]
1343mod tests {
1344    use super::*;
1345    use nu_utils::SharedCow;
1346
1347    fn test_chunks<T>(data: Vec<T>, type_: ByteStreamType) -> Chunks
1348    where
1349        T: AsRef<[u8]> + Default + Send + 'static,
1350    {
1351        let reader = ReadIterator {
1352            iter: data.into_iter(),
1353            cursor: Some(Cursor::new(T::default())),
1354        };
1355        Chunks::new(
1356            SourceReader::Read(Box::new(reader)),
1357            Span::test_data(),
1358            Signals::empty(),
1359            type_,
1360        )
1361    }
1362
1363    #[test]
1364    fn chunks_read_binary_passthrough() {
1365        let bins = vec![&[0, 1][..], &[2, 3][..]];
1366        let iter = test_chunks(bins.clone(), ByteStreamType::Binary);
1367
1368        let bins_values: Vec<Value> = bins
1369            .into_iter()
1370            .map(|bin| Value::binary(bin, Span::test_data()))
1371            .collect();
1372        assert_eq!(
1373            bins_values,
1374            iter.collect::<Result<Vec<Value>, _>>().expect("error")
1375        );
1376    }
1377
1378    #[test]
1379    fn read_binary_shares_data() {
1380        let data = SharedCow::new(vec![0, 1, 2, 3]);
1381        let stream = ByteStream::read_binary(data.clone(), Span::test_data(), Signals::empty());
1382
1383        assert_eq!(SharedCow::ref_count(&data), 2);
1384        assert_eq!(stream.into_bytes().expect("error"), data.as_slice());
1385    }
1386
1387    #[test]
1388    fn chunks_read_string_clean() {
1389        let strs = vec!["Nushell", "が好きです"];
1390        let iter = test_chunks(strs.clone(), ByteStreamType::String);
1391
1392        let strs_values: Vec<Value> = strs
1393            .into_iter()
1394            .map(|string| Value::string(string, Span::test_data()))
1395            .collect();
1396        assert_eq!(
1397            strs_values,
1398            iter.collect::<Result<Vec<Value>, _>>().expect("error")
1399        );
1400    }
1401
1402    #[test]
1403    fn chunks_read_string_split_boundary() {
1404        let real = "Nushell最高!";
1405        let chunks = vec![&b"Nushell\xe6"[..], &b"\x9c\x80\xe9"[..], &b"\xab\x98!"[..]];
1406        let iter = test_chunks(chunks.clone(), ByteStreamType::String);
1407
1408        let mut string = String::new();
1409        for value in iter {
1410            let chunk_string = value.expect("error").into_string().expect("not a string");
1411            string.push_str(&chunk_string);
1412        }
1413        assert_eq!(real, string);
1414    }
1415
1416    #[test]
1417    fn chunks_read_string_utf8_error() {
1418        let chunks = vec![&b"Nushell\xe6"[..], &b"\x9c\x80\xe9"[..], &b"\xab"[..]];
1419        let iter = test_chunks(chunks, ByteStreamType::String);
1420
1421        let mut string = String::new();
1422        for value in iter {
1423            match value {
1424                Ok(value) => string.push_str(&value.into_string().expect("not a string")),
1425                Err(err) => {
1426                    println!("string so far: {string:?}");
1427                    println!("got error: {err:?}");
1428                    assert!(!string.is_empty());
1429                    assert!(matches!(err, ShellError::NonUtf8Custom { .. }));
1430                    return;
1431                }
1432            }
1433        }
1434        panic!("no error");
1435    }
1436
1437    #[test]
1438    fn chunks_read_unknown_fallback() {
1439        let chunks = vec![&b"Nushell"[..], &b"\x9c\x80\xe9abcd"[..], &b"efgh"[..]];
1440        let mut iter = test_chunks(chunks, ByteStreamType::Unknown);
1441
1442        let mut get = || iter.next().expect("end of iter").expect("error");
1443
1444        assert_eq!(Value::test_string("Nushell"), get());
1445        assert_eq!(Value::test_binary(b"\x9c\x80\xe9abcd"), get());
1446        // Once it's in binary mode it won't go back
1447        assert_eq!(Value::test_binary(b"efgh"), get());
1448    }
1449}