1#[cfg(feature = "os")]
6use crate::process::{ChildPipe, ChildProcess};
7use crate::{
8 IntRange, PipelineData, ShellError, Signals, Span, Type, Value,
9 shell_error::{bridge::ShellErrorBridge, io::IoError},
10};
11use nu_utils::SplitRead as SplitReadInner;
12use serde::{Deserialize, Serialize};
13use std::ops::Bound;
14#[cfg(unix)]
15use std::os::fd::OwnedFd;
16#[cfg(windows)]
17use std::os::windows::io::OwnedHandle;
18use std::{
19 fmt::Debug,
20 fs::File,
21 io::{self, BufRead, BufReader, Cursor, ErrorKind, Read, Write},
22 process::Stdio,
23};
24
25pub enum ByteStreamSource {
32 Read(Box<dyn Read + Send + 'static>),
33 File(File),
34 #[cfg(feature = "os")]
35 Child(Box<ChildProcess>),
36}
37
38impl ByteStreamSource {
39 fn reader(self) -> Option<SourceReader> {
40 match self {
41 ByteStreamSource::Read(read) => Some(SourceReader::Read(read)),
42 ByteStreamSource::File(file) => Some(SourceReader::File(file)),
43 #[cfg(feature = "os")]
44 ByteStreamSource::Child(mut child) => child.stdout.take().map(|stdout| match stdout {
45 ChildPipe::Pipe(pipe) => SourceReader::File(convert_file(pipe)),
46 ChildPipe::Tee(tee) => SourceReader::Read(tee),
47 }),
48 }
49 }
50
51 #[cfg(feature = "os")]
53 pub fn is_external(&self) -> bool {
54 matches!(self, ByteStreamSource::Child(..))
55 }
56
57 #[cfg(not(feature = "os"))]
58 pub fn is_external(&self) -> bool {
59 false
61 }
62}
63
64impl Debug for ByteStreamSource {
65 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66 match self {
67 ByteStreamSource::Read(_) => f.debug_tuple("Read").field(&"..").finish(),
68 ByteStreamSource::File(file) => f.debug_tuple("File").field(file).finish(),
69 #[cfg(feature = "os")]
70 ByteStreamSource::Child(child) => f.debug_tuple("Child").field(child).finish(),
71 }
72 }
73}
74
75enum SourceReader {
76 Read(Box<dyn Read + Send + 'static>),
77 File(File),
78}
79
80impl Read for SourceReader {
81 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
82 match self {
83 SourceReader::Read(reader) => reader.read(buf),
84 SourceReader::File(file) => file.read(buf),
85 }
86 }
87}
88
89impl Debug for SourceReader {
90 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
91 match self {
92 SourceReader::Read(_) => f.debug_tuple("Read").field(&"..").finish(),
93 SourceReader::File(file) => f.debug_tuple("File").field(file).finish(),
94 }
95 }
96}
97
98#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
100pub enum ByteStreamType {
101 Binary,
104 String,
110 #[default]
113 Unknown,
114}
115
116impl ByteStreamType {
117 pub fn describe(self) -> &'static str {
120 match self {
121 ByteStreamType::Binary => "binary (stream)",
122 ByteStreamType::String => "string (stream)",
123 ByteStreamType::Unknown => "byte stream",
124 }
125 }
126
127 pub fn is_binary_coercible(self) -> bool {
129 matches!(self, ByteStreamType::Binary | ByteStreamType::Unknown)
130 }
131
132 pub fn is_string_coercible(self) -> bool {
134 matches!(self, ByteStreamType::String | ByteStreamType::Unknown)
135 }
136}
137
138impl From<ByteStreamType> for Type {
139 fn from(value: ByteStreamType) -> Self {
140 match value {
141 ByteStreamType::Binary => Type::Binary,
142 ByteStreamType::String => Type::String,
143 ByteStreamType::Unknown => Type::Any,
144 }
145 }
146}
147
148#[derive(Debug)]
191pub struct ByteStream {
192 stream: ByteStreamSource,
193 span: Span,
194 signals: Signals,
195 type_: ByteStreamType,
196 known_size: Option<u64>,
197 caller_spans: Vec<Span>,
198}
199
200impl ByteStream {
201 pub fn new(
203 stream: ByteStreamSource,
204 span: Span,
205 signals: Signals,
206 type_: ByteStreamType,
207 ) -> Self {
208 Self {
209 stream,
210 span,
211 signals,
212 type_,
213 known_size: None,
214 caller_spans: vec![],
215 }
216 }
217
218 pub fn push_caller_span(&mut self, span: Span) {
220 if span != self.span {
221 self.caller_spans.push(span)
222 }
223 }
224
225 pub fn get_caller_spans(&self) -> &Vec<Span> {
227 &self.caller_spans
228 }
229
230 pub fn read(
232 reader: impl Read + Send + 'static,
233 span: Span,
234 signals: Signals,
235 type_: ByteStreamType,
236 ) -> Self {
237 Self::new(
238 ByteStreamSource::Read(Box::new(reader)),
239 span,
240 signals,
241 type_,
242 )
243 }
244
245 pub fn skip(self, span: Span, n: u64) -> Result<Self, ShellError> {
246 let known_size = self.known_size.map(|len| len.saturating_sub(n));
247 if let Some(mut reader) = self.reader() {
248 io::copy(&mut (&mut reader).take(n), &mut io::sink())
250 .map_err(|err| IoError::new(err, span, None))?;
251 Ok(
252 ByteStream::read(reader, span, Signals::empty(), ByteStreamType::Binary)
253 .with_known_size(known_size),
254 )
255 } else {
256 Err(ShellError::TypeMismatch {
257 err_message: "expected readable stream".into(),
258 span,
259 })
260 }
261 }
262
263 pub fn take(self, span: Span, n: u64) -> Result<Self, ShellError> {
264 let known_size = self.known_size.map(|s| s.min(n));
265 if let Some(reader) = self.reader() {
266 Ok(ByteStream::read(
267 reader.take(n),
268 span,
269 Signals::empty(),
270 ByteStreamType::Binary,
271 )
272 .with_known_size(known_size))
273 } else {
274 Err(ShellError::TypeMismatch {
275 err_message: "expected readable stream".into(),
276 span,
277 })
278 }
279 }
280
281 pub fn slice(
282 self,
283 val_span: Span,
284 call_span: Span,
285 range: IntRange,
286 ) -> Result<Self, ShellError> {
287 if let Some(len) = self.known_size {
288 let start = range.absolute_start(len);
289 let stream = self.skip(val_span, start);
290
291 match range.absolute_end(len) {
292 Bound::Unbounded => stream,
293 Bound::Included(end) | Bound::Excluded(end) if end < start => {
294 stream.and_then(|s| s.take(val_span, 0))
295 }
296 Bound::Included(end) => {
297 let distance = end - start + 1;
298 stream.and_then(|s| s.take(val_span, distance.min(len)))
299 }
300 Bound::Excluded(end) => {
301 let distance = end - start;
302 stream.and_then(|s| s.take(val_span, distance.min(len)))
303 }
304 }
305 } else if range.is_relative() {
306 Err(ShellError::RelativeRangeOnInfiniteStream { span: call_span })
307 } else {
308 let start = range.start() as u64;
309 let stream = self.skip(val_span, start);
310
311 match range.distance() {
312 Bound::Unbounded => stream,
313 Bound::Included(distance) => stream.and_then(|s| s.take(val_span, distance + 1)),
314 Bound::Excluded(distance) => stream.and_then(|s| s.take(val_span, distance)),
315 }
316 }
317 }
318
319 pub fn read_string(string: String, span: Span, signals: Signals) -> Self {
321 let len = string.len();
322 ByteStream::read(
323 Cursor::new(string.into_bytes()),
324 span,
325 signals,
326 ByteStreamType::String,
327 )
328 .with_known_size(Some(len as u64))
329 }
330
331 pub fn read_binary<T>(bytes: T, span: Span, signals: Signals) -> Self
333 where
334 T: AsRef<[u8]> + Send + 'static,
335 {
336 let len = bytes.as_ref().len();
337 ByteStream::read(Cursor::new(bytes), span, signals, ByteStreamType::Binary)
338 .with_known_size(Some(len as u64))
339 }
340
341 pub fn file(file: File, span: Span, signals: Signals) -> Self {
346 Self::new(
347 ByteStreamSource::File(file),
348 span,
349 signals,
350 ByteStreamType::Unknown,
351 )
352 }
353
354 #[cfg(feature = "os")]
359 pub fn child(child: ChildProcess, span: Span) -> Self {
360 Self::new(
361 ByteStreamSource::Child(Box::new(child)),
362 span,
363 Signals::empty(),
364 ByteStreamType::Unknown,
365 )
366 }
367
368 #[cfg(feature = "os")]
373 pub fn stdin(span: Span) -> Result<Self, ShellError> {
374 let stdin = os_pipe::dup_stdin().map_err(|err| IoError::new(err, span, None))?;
375 let source = ByteStreamSource::File(convert_file(stdin));
376 Ok(Self::new(
377 source,
378 span,
379 Signals::empty(),
380 ByteStreamType::Unknown,
381 ))
382 }
383
384 #[cfg(not(feature = "os"))]
385 pub fn stdin(span: Span) -> Result<Self, ShellError> {
386 Err(ShellError::DisabledOsSupport {
387 msg: "Stdin is not supported".to_string(),
388 span,
389 })
390 }
391
392 pub fn from_fn(
395 span: Span,
396 signals: Signals,
397 type_: ByteStreamType,
398 generator: impl FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
399 ) -> Self {
400 Self::read(
401 ReadGenerator {
402 buffer: Cursor::new(Vec::new()),
403 generator,
404 },
405 span,
406 signals,
407 type_,
408 )
409 }
410
411 pub fn with_type(mut self, type_: ByteStreamType) -> Self {
412 self.type_ = type_;
413 self
414 }
415
416 pub fn from_iter<I>(iter: I, span: Span, signals: Signals, type_: ByteStreamType) -> Self
420 where
421 I: IntoIterator,
422 I::IntoIter: Send + 'static,
423 I::Item: AsRef<[u8]> + Default + Send + 'static,
424 {
425 let iter = iter.into_iter();
426 let cursor = Some(Cursor::new(I::Item::default()));
427 Self::read(ReadIterator { iter, cursor }, span, signals, type_)
428 }
429
430 pub fn from_result_iter<I, T>(
434 iter: I,
435 span: Span,
436 signals: Signals,
437 type_: ByteStreamType,
438 ) -> Self
439 where
440 I: IntoIterator<Item = Result<T, ShellError>>,
441 I::IntoIter: Send + 'static,
442 T: AsRef<[u8]> + Default + Send + 'static,
443 {
444 let iter = iter.into_iter();
445 let cursor = Some(Cursor::new(T::default()));
446 Self::read(ReadResultIterator { iter, cursor }, span, signals, type_)
447 }
448
449 pub fn with_known_size(mut self, size: Option<u64>) -> Self {
451 self.known_size = size;
452 self
453 }
454
455 pub fn source(&self) -> &ByteStreamSource {
457 &self.stream
458 }
459
460 pub fn source_mut(&mut self) -> &mut ByteStreamSource {
462 &mut self.stream
463 }
464
465 pub fn span(&self) -> Span {
467 self.span
468 }
469
470 pub fn with_span(mut self, span: Span) -> Self {
472 self.span = span;
473 self
474 }
475
476 pub fn signals(&self) -> &Signals {
479 &self.signals
480 }
481
482 pub fn type_(&self) -> ByteStreamType {
483 self.type_
484 }
485
486 pub fn known_size(&self) -> Option<u64> {
488 self.known_size
489 }
490
491 pub fn reader(self) -> Option<Reader> {
498 let reader = self.stream.reader()?;
499 Some(Reader {
500 reader: BufReader::new(reader),
501 span: self.span,
502 signals: self.signals,
503 })
504 }
505
506 pub fn lines(self) -> Option<Lines> {
514 let reader = self.stream.reader()?;
515 Some(Lines {
516 reader: BufReader::new(reader),
517 span: self.span,
518 signals: self.signals,
519 strict: false,
520 })
521 }
522
523 pub fn split(self, delimiter: Vec<u8>) -> Option<SplitRead> {
531 let reader = self.stream.reader()?;
532 Some(SplitRead::new(reader, delimiter, self.span, self.signals))
533 }
534
535 pub fn chunks(self) -> Option<Chunks> {
551 let reader = self.stream.reader()?;
552 Some(Chunks::new(reader, self.span, self.signals, self.type_))
553 }
554
555 pub fn into_source(self) -> ByteStreamSource {
557 self.stream
558 }
559
560 pub fn into_stdio(mut self) -> Result<Stdio, Self> {
568 match self.stream {
569 ByteStreamSource::Read(..) => Err(self),
570 ByteStreamSource::File(file) => Ok(file.into()),
571 #[cfg(feature = "os")]
572 ByteStreamSource::Child(child) => {
573 if let ChildProcess {
574 stdout: Some(ChildPipe::Pipe(stdout)),
575 stderr,
576 ..
577 } = *child
578 {
579 debug_assert!(stderr.is_none(), "stderr should not exist");
580 Ok(stdout.into())
581 } else {
582 self.stream = ByteStreamSource::Child(child);
583 Err(self)
584 }
585 }
586 }
587 }
588
589 #[cfg(feature = "os")]
594 pub fn into_child(self) -> Result<ChildProcess, Self> {
595 if let ByteStreamSource::Child(child) = self.stream {
596 Ok(*child)
597 } else {
598 Err(self)
599 }
600 }
601
602 pub fn into_bytes(self) -> Result<Vec<u8>, ShellError> {
606 let span = self.span;
607 let signals = self.signals;
608 let from_io_error = IoError::factory(span, None);
609 match self.stream {
610 ByteStreamSource::Read(mut read) => {
611 let mut buf = Vec::new();
612 let mut chunk = [0; DEFAULT_BUF_SIZE];
613 loop {
614 signals.check(&span)?;
615 match read.read(&mut chunk) {
616 Ok(0) => break,
617 Ok(n) => buf.extend_from_slice(&chunk[..n]),
618 Err(e) if e.kind() == ErrorKind::Interrupted => continue,
619 Err(e) => match ShellErrorBridge::try_from(e) {
620 Ok(ShellErrorBridge(e)) => return Err(e),
621 Err(e) => return Err(ShellError::Io(from_io_error(e))),
622 },
623 }
624 }
625 Ok(buf)
626 }
627 ByteStreamSource::File(mut file) => {
628 let mut buf = Vec::new();
629 let mut chunk = [0; DEFAULT_BUF_SIZE];
630 loop {
631 signals.check(&span)?;
632 match file.read(&mut chunk) {
633 Ok(0) => break,
634 Ok(n) => buf.extend_from_slice(&chunk[..n]),
635 Err(e) if e.kind() == ErrorKind::Interrupted => continue,
636 Err(e) => return Err(ShellError::Io(from_io_error(e))),
637 }
638 }
639 Ok(buf)
640 }
641 #[cfg(feature = "os")]
642 ByteStreamSource::Child(child) => child.into_bytes(),
643 }
644 }
645
646 pub fn into_string(self) -> Result<String, ShellError> {
655 let span = self.span;
656 if self.type_.is_string_coercible() {
657 let trim = self.stream.is_external();
658 let bytes = self.into_bytes()?;
659 let mut string = String::from_utf8(bytes).map_err(|err| ShellError::NonUtf8Custom {
660 span,
661 msg: err.to_string(),
662 })?;
663 if trim {
664 trim_end_newline(&mut string);
665 }
666 Ok(string)
667 } else {
668 Err(ShellError::TypeMismatch {
669 err_message: "expected string, but got binary".into(),
670 span,
671 })
672 }
673 }
674
675 pub fn into_value(self) -> Result<Value, ShellError> {
688 let span = self.span;
689 let trim = self.stream.is_external();
690 let value = match self.type_ {
691 ByteStreamType::Binary => Value::binary(self.into_bytes()?, span),
693 ByteStreamType::String => Value::string(self.into_string()?, span),
695 ByteStreamType::Unknown => {
697 value_from_bytes(self.into_bytes()?, span, ByteStreamType::Unknown, trim)
698 }
699 };
700 Ok(value)
701 }
702
703 pub fn drain(self) -> Result<(), ShellError> {
705 match self.stream {
706 ByteStreamSource::Read(read) => {
707 copy_with_signals(read, io::sink(), self.span, &self.signals)?;
708 Ok(())
709 }
710 ByteStreamSource::File(_) => Ok(()),
711 #[cfg(feature = "os")]
712 ByteStreamSource::Child(child) => child.wait(),
713 }
714 }
715
716 pub fn print(self, to_stderr: bool) -> Result<(), ShellError> {
718 if to_stderr {
719 self.write_to(&mut io::stderr())
720 } else {
721 self.write_to(&mut io::stdout())
722 }
723 }
724
725 pub fn write_to(self, dest: impl Write) -> Result<(), ShellError> {
727 let span = self.span;
728 let signals = &self.signals;
729 match self.stream {
730 ByteStreamSource::Read(read) => {
731 copy_with_signals(read, dest, span, signals)?;
732 }
733 ByteStreamSource::File(file) => {
734 copy_with_signals(file, dest, span, signals)?;
735 }
736 #[cfg(feature = "os")]
737 ByteStreamSource::Child(mut child) => {
738 debug_assert!(child.stderr.is_none(), "stderr should not exist");
742
743 if let Some(stdout) = child.stdout.take() {
744 match stdout {
745 ChildPipe::Pipe(pipe) => {
746 copy_with_signals(pipe, dest, span, signals)?;
747 }
748 ChildPipe::Tee(tee) => {
749 copy_with_signals(tee, dest, span, signals)?;
750 }
751 }
752 }
753 child.wait()?;
754 }
755 }
756 Ok(())
757 }
758
759 #[cfg(feature = "os")]
762 pub fn child_with_separate_stderr(&mut self) -> Option<&mut ChildProcess> {
763 match &mut self.stream {
764 ByteStreamSource::Child(child) if child.stderr.is_some() => Some(child),
765 _ => None,
766 }
767 }
768
769 #[cfg(feature = "os")]
773 pub fn copy_stdout(&mut self) -> Result<Option<Self>, ShellError> {
774 let (span, signals, type_) = (self.span, self.signals.clone(), self.type_);
775 let Some(child) = self.child_with_separate_stderr() else {
776 return Ok(None);
777 };
778 let copy = ByteStreamSource::Child(Box::new(child.copy_stdout()?));
779 Ok(Some(Self::new(copy, span, signals, type_)))
780 }
781
782 #[cfg(feature = "os")]
786 pub fn write_stdout_to(mut self, dest: impl Write) -> Result<Option<Self>, ShellError> {
787 let ByteStreamSource::Child(child) = &mut self.stream else {
788 return self.write_to(dest).map(|()| None);
789 };
790 if child.stderr.is_none() {
791 return self.write_to(dest).map(|()| None);
792 }
793 if child.stdout.is_some() {
794 child.buffer_pipes()?;
796 if let Some(stdout) = child.stdout.take() {
797 copy_with_signals(stdout, dest, self.span, &self.signals)?;
798 }
799 }
800 Ok(Some(self))
801 }
802}
803
804impl From<ByteStream> for PipelineData {
805 fn from(stream: ByteStream) -> Self {
806 Self::byte_stream(stream, None)
807 }
808}
809
810struct ReadIterator<I>
811where
812 I: Iterator,
813 I::Item: AsRef<[u8]>,
814{
815 iter: I,
816 cursor: Option<Cursor<I::Item>>,
817}
818
819impl<I> Read for ReadIterator<I>
820where
821 I: Iterator,
822 I::Item: AsRef<[u8]>,
823{
824 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
825 while let Some(cursor) = self.cursor.as_mut() {
826 let read = cursor.read(buf)?;
827 if read == 0 {
828 self.cursor = self.iter.next().map(Cursor::new);
829 } else {
830 return Ok(read);
831 }
832 }
833 Ok(0)
834 }
835}
836
837struct ReadResultIterator<I, T>
838where
839 I: Iterator<Item = Result<T, ShellError>>,
840 T: AsRef<[u8]>,
841{
842 iter: I,
843 cursor: Option<Cursor<T>>,
844}
845
846impl<I, T> Read for ReadResultIterator<I, T>
847where
848 I: Iterator<Item = Result<T, ShellError>>,
849 T: AsRef<[u8]>,
850{
851 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
852 while let Some(cursor) = self.cursor.as_mut() {
853 let read = cursor.read(buf)?;
854 if read == 0 {
855 self.cursor = self
856 .iter
857 .next()
858 .transpose()
859 .map_err(ShellErrorBridge)?
860 .map(Cursor::new);
861 } else {
862 return Ok(read);
863 }
864 }
865 Ok(0)
866 }
867}
868
869pub struct Reader {
870 reader: BufReader<SourceReader>,
871 span: Span,
872 signals: Signals,
873}
874
875impl Reader {
876 pub fn span(&self) -> Span {
877 self.span
878 }
879}
880
881impl Read for Reader {
882 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
883 self.signals.check(&self.span).map_err(ShellErrorBridge)?;
884 self.reader.read(buf)
885 }
886}
887
888impl BufRead for Reader {
889 fn fill_buf(&mut self) -> io::Result<&[u8]> {
890 self.reader.fill_buf()
891 }
892
893 fn consume(&mut self, amt: usize) {
894 self.reader.consume(amt)
895 }
896}
897
898pub struct Lines {
899 reader: BufReader<SourceReader>,
900 span: Span,
901 signals: Signals,
902 strict: bool,
911}
912
913impl Lines {
914 pub fn span(&self) -> Span {
915 self.span
916 }
917
918 pub fn strict(mut self, strict: bool) -> Self {
924 self.strict = strict;
925 self
926 }
927}
928
929impl Iterator for Lines {
930 type Item = Result<String, ShellError>;
931
932 fn next(&mut self) -> Option<Self::Item> {
933 if self.signals.interrupted() {
934 None
935 } else {
936 let mut buf = Vec::new();
937 match self.reader.read_until(b'\n', &mut buf) {
938 Ok(0) => None,
939 Ok(_) => {
940 let mut string = if self.strict {
941 match String::from_utf8(buf) {
942 Ok(s) => s,
943 Err(_) => return Some(Err(ShellError::NonUtf8 { span: self.span })),
944 }
945 } else {
946 String::from_utf8_lossy(&buf).into_owned()
947 };
948 trim_end_newline(&mut string);
949 Some(Ok(string))
950 }
951 Err(err) => Some(Err(IoError::new(err, self.span, None).into())),
952 }
953 }
954 }
955}
956
957pub struct SplitRead {
958 internal: SplitReadInner<BufReader<SourceReader>>,
959 span: Span,
960 signals: Signals,
961}
962
963impl SplitRead {
964 fn new(
965 reader: SourceReader,
966 delimiter: impl AsRef<[u8]>,
967 span: Span,
968 signals: Signals,
969 ) -> Self {
970 Self {
971 internal: SplitReadInner::new(BufReader::new(reader), delimiter),
972 span,
973 signals,
974 }
975 }
976
977 pub fn span(&self) -> Span {
978 self.span
979 }
980}
981
982impl Iterator for SplitRead {
983 type Item = Result<Vec<u8>, ShellError>;
984
985 fn next(&mut self) -> Option<Self::Item> {
986 if self.signals.interrupted() {
987 return None;
988 }
989 self.internal.next().map(|r| {
990 r.map_err(|err| {
991 ShellError::Io(IoError::new_internal(
992 err,
993 "Could not get next value for SplitRead",
994 ))
995 })
996 })
997 }
998}
999
1000pub struct Chunks {
1006 reader: BufReader<SourceReader>,
1007 pos: u64,
1008 error: bool,
1009 span: Span,
1010 signals: Signals,
1011 type_: ByteStreamType,
1012}
1013
1014impl Chunks {
1015 fn new(reader: SourceReader, span: Span, signals: Signals, type_: ByteStreamType) -> Self {
1016 Self {
1017 reader: BufReader::new(reader),
1018 pos: 0,
1019 error: false,
1020 span,
1021 signals,
1022 type_,
1023 }
1024 }
1025
1026 pub fn span(&self) -> Span {
1027 self.span
1028 }
1029
1030 fn next_string(&mut self) -> Result<Option<String>, (Vec<u8>, ShellError)> {
1031 let from_io_error = |err: std::io::Error| match ShellErrorBridge::try_from(err) {
1032 Ok(err) => err.0,
1033 Err(err) => IoError::new(err, self.span, None).into(),
1034 };
1035
1036 let buf = self
1038 .reader
1039 .fill_buf()
1040 .map_err(from_io_error)
1041 .map_err(|err| (vec![], err))?;
1042
1043 if buf.is_empty() {
1045 return Ok(None);
1046 }
1047
1048 let mut buf = buf.to_vec();
1049 let mut consumed = 0;
1050
1051 if buf.len() < 4 {
1053 consumed += buf.len();
1054 self.reader.consume(buf.len());
1055 match self.reader.fill_buf() {
1056 Ok(more_bytes) => buf.extend_from_slice(more_bytes),
1057 Err(err) => return Err((buf, from_io_error(err))),
1058 }
1059 }
1060
1061 match String::from_utf8(buf) {
1063 Ok(string) => {
1064 self.reader.consume(string.len() - consumed);
1065 self.pos += string.len() as u64;
1066 Ok(Some(string))
1067 }
1068 Err(err) if err.utf8_error().error_len().is_none() => {
1069 let valid_up_to = err.utf8_error().valid_up_to();
1072 if valid_up_to > consumed {
1073 self.reader.consume(valid_up_to - consumed);
1074 }
1075 let mut buf = err.into_bytes();
1076 buf.truncate(valid_up_to);
1077 buf.shrink_to_fit();
1078 let string = String::from_utf8(buf)
1079 .expect("failed to parse utf-8 even after correcting error");
1080 self.pos += string.len() as u64;
1081 Ok(Some(string))
1082 }
1083 Err(err) => {
1084 let shell_error = ShellError::NonUtf8Custom {
1086 msg: format!("invalid utf-8 sequence starting at index {}", self.pos),
1087 span: self.span,
1088 };
1089 let buf = err.into_bytes();
1090 if buf.len() > consumed {
1093 self.reader.consume(buf.len() - consumed);
1094 }
1095 self.pos += buf.len() as u64;
1096 Err((buf, shell_error))
1097 }
1098 }
1099 }
1100}
1101
1102impl Iterator for Chunks {
1103 type Item = Result<Value, ShellError>;
1104
1105 fn next(&mut self) -> Option<Self::Item> {
1106 if self.error || self.signals.interrupted() {
1107 None
1108 } else {
1109 match self.type_ {
1110 ByteStreamType::Binary => {
1112 let buf = match self.reader.fill_buf() {
1113 Ok(buf) => buf,
1114 Err(err) => {
1115 self.error = true;
1116 return Some(Err(ShellError::Io(IoError::new(err, self.span, None))));
1117 }
1118 };
1119 if !buf.is_empty() {
1120 let len = buf.len();
1121 let value = Value::binary(buf, self.span);
1122 self.reader.consume(len);
1123 self.pos += len as u64;
1124 Some(Ok(value))
1125 } else {
1126 None
1127 }
1128 }
1129 ByteStreamType::String => match self.next_string().transpose()? {
1131 Ok(string) => Some(Ok(Value::string(string, self.span))),
1132 Err((_, err)) => {
1133 self.error = true;
1134 Some(Err(err))
1135 }
1136 },
1137 ByteStreamType::Unknown => {
1140 match self.next_string().transpose()? {
1141 Ok(string) => Some(Ok(Value::string(string, self.span))),
1142 Err((buf, _)) if !buf.is_empty() => {
1143 self.type_ = ByteStreamType::Binary;
1145 Some(Ok(Value::binary(buf, self.span)))
1146 }
1147 Err((_, err)) => {
1148 self.error = true;
1149 Some(Err(err))
1150 }
1151 }
1152 }
1153 }
1154 }
1155 }
1156}
1157
1158fn trim_end_newline(string: &mut String) {
1159 if string.ends_with('\n') {
1160 string.pop();
1161 if string.ends_with('\r') {
1162 string.pop();
1163 }
1164 }
1165}
1166
1167pub fn value_from_bytes(
1181 bytes: Vec<u8>,
1182 span: Span,
1183 type_: ByteStreamType,
1184 trim_trailing_newline: bool,
1185) -> Value {
1186 if matches!(type_, ByteStreamType::Binary) {
1187 return Value::binary(bytes, span);
1188 }
1189
1190 match String::from_utf8(bytes) {
1191 Ok(mut s) => {
1192 if trim_trailing_newline {
1193 trim_end_newline(&mut s);
1194 }
1195 Value::string(s, span)
1196 }
1197 Err(err) => {
1198 let valid_up_to = err.utf8_error().valid_up_to();
1199 let incomplete_at_end = err.utf8_error().error_len().is_none();
1201 let bytes = err.into_bytes();
1202 if incomplete_at_end && valid_up_to > 0 {
1203 let mut s = String::from_utf8(bytes[..valid_up_to].to_vec())
1205 .expect("valid_up_to marks a valid UTF-8 prefix");
1206 if trim_trailing_newline {
1207 trim_end_newline(&mut s);
1208 }
1209 Value::string(s, span)
1210 } else {
1211 Value::binary(bytes, span)
1212 }
1213 }
1214 }
1215}
1216
1217#[cfg(unix)]
1218pub(crate) fn convert_file<T: From<OwnedFd>>(file: impl Into<OwnedFd>) -> T {
1219 file.into().into()
1220}
1221
1222#[cfg(windows)]
1223pub(crate) fn convert_file<T: From<OwnedHandle>>(file: impl Into<OwnedHandle>) -> T {
1224 file.into().into()
1225}
1226
1227const DEFAULT_BUF_SIZE: usize = 8192;
1228
1229pub fn copy_with_signals(
1230 mut reader: impl Read,
1231 mut writer: impl Write,
1232 span: Span,
1233 signals: &Signals,
1234) -> Result<u64, ShellError> {
1235 let from_io_error = IoError::factory(span, None);
1236 if signals.is_empty() {
1237 match io::copy(&mut reader, &mut writer) {
1238 Ok(n) => {
1239 writer.flush().map_err(&from_io_error)?;
1240 Ok(n)
1241 }
1242 Err(err) => {
1243 let _ = writer.flush();
1244 match ShellErrorBridge::try_from(err) {
1245 Ok(ShellErrorBridge(shell_error)) => Err(shell_error),
1246 Err(err) => Err(from_io_error(err).into()),
1247 }
1248 }
1249 }
1250 } else {
1251 match generic_copy(&mut reader, &mut writer, span, signals) {
1256 Ok(len) => {
1257 writer.flush().map_err(&from_io_error)?;
1258 Ok(len)
1259 }
1260 Err(err) => {
1261 let _ = writer.flush();
1262 Err(err)
1263 }
1264 }
1265 }
1266}
1267
1268fn generic_copy(
1270 mut reader: impl Read,
1271 mut writer: impl Write,
1272 span: Span,
1273 signals: &Signals,
1274) -> Result<u64, ShellError> {
1275 let from_io_error = IoError::factory(span, None);
1276 let buf = &mut [0; DEFAULT_BUF_SIZE];
1277 let mut len = 0;
1278 loop {
1279 signals.check(&span)?;
1280 let n = match reader.read(buf) {
1281 Ok(0) => break,
1282 Ok(n) => n,
1283 Err(e) if e.kind() == ErrorKind::Interrupted => continue,
1284 Err(e) => match ShellErrorBridge::try_from(e) {
1285 Ok(ShellErrorBridge(e)) => return Err(e),
1286 Err(e) => return Err(from_io_error(e).into()),
1287 },
1288 };
1289 len += n;
1290 writer.write_all(&buf[..n]).map_err(&from_io_error)?;
1291 }
1292 Ok(len as u64)
1293}
1294
1295struct ReadGenerator<F>
1296where
1297 F: FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
1298{
1299 buffer: Cursor<Vec<u8>>,
1300 generator: F,
1301}
1302
1303impl<F> BufRead for ReadGenerator<F>
1304where
1305 F: FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
1306{
1307 fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
1308 while self.buffer.fill_buf()?.is_empty() {
1311 self.buffer.set_position(0);
1313 self.buffer.get_mut().clear();
1314 if !(self.generator)(self.buffer.get_mut()).map_err(ShellErrorBridge)? {
1316 break;
1318 }
1319 }
1320 self.buffer.fill_buf()
1321 }
1322
1323 fn consume(&mut self, amt: usize) {
1324 self.buffer.consume(amt);
1325 }
1326}
1327
1328impl<F> Read for ReadGenerator<F>
1329where
1330 F: FnMut(&mut Vec<u8>) -> Result<bool, ShellError> + Send + 'static,
1331{
1332 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1333 let slice = self.fill_buf()?;
1335 let len = buf.len().min(slice.len());
1336 buf[..len].copy_from_slice(&slice[..len]);
1337 self.consume(len);
1338 Ok(len)
1339 }
1340}
1341
1342#[cfg(test)]
1343mod tests {
1344 use super::*;
1345 use nu_utils::SharedCow;
1346
1347 fn test_chunks<T>(data: Vec<T>, type_: ByteStreamType) -> Chunks
1348 where
1349 T: AsRef<[u8]> + Default + Send + 'static,
1350 {
1351 let reader = ReadIterator {
1352 iter: data.into_iter(),
1353 cursor: Some(Cursor::new(T::default())),
1354 };
1355 Chunks::new(
1356 SourceReader::Read(Box::new(reader)),
1357 Span::test_data(),
1358 Signals::empty(),
1359 type_,
1360 )
1361 }
1362
1363 #[test]
1364 fn chunks_read_binary_passthrough() {
1365 let bins = vec![&[0, 1][..], &[2, 3][..]];
1366 let iter = test_chunks(bins.clone(), ByteStreamType::Binary);
1367
1368 let bins_values: Vec<Value> = bins
1369 .into_iter()
1370 .map(|bin| Value::binary(bin, Span::test_data()))
1371 .collect();
1372 assert_eq!(
1373 bins_values,
1374 iter.collect::<Result<Vec<Value>, _>>().expect("error")
1375 );
1376 }
1377
1378 #[test]
1379 fn read_binary_shares_data() {
1380 let data = SharedCow::new(vec![0, 1, 2, 3]);
1381 let stream = ByteStream::read_binary(data.clone(), Span::test_data(), Signals::empty());
1382
1383 assert_eq!(SharedCow::ref_count(&data), 2);
1384 assert_eq!(stream.into_bytes().expect("error"), data.as_slice());
1385 }
1386
1387 #[test]
1388 fn chunks_read_string_clean() {
1389 let strs = vec!["Nushell", "が好きです"];
1390 let iter = test_chunks(strs.clone(), ByteStreamType::String);
1391
1392 let strs_values: Vec<Value> = strs
1393 .into_iter()
1394 .map(|string| Value::string(string, Span::test_data()))
1395 .collect();
1396 assert_eq!(
1397 strs_values,
1398 iter.collect::<Result<Vec<Value>, _>>().expect("error")
1399 );
1400 }
1401
1402 #[test]
1403 fn chunks_read_string_split_boundary() {
1404 let real = "Nushell最高!";
1405 let chunks = vec![&b"Nushell\xe6"[..], &b"\x9c\x80\xe9"[..], &b"\xab\x98!"[..]];
1406 let iter = test_chunks(chunks.clone(), ByteStreamType::String);
1407
1408 let mut string = String::new();
1409 for value in iter {
1410 let chunk_string = value.expect("error").into_string().expect("not a string");
1411 string.push_str(&chunk_string);
1412 }
1413 assert_eq!(real, string);
1414 }
1415
1416 #[test]
1417 fn chunks_read_string_utf8_error() {
1418 let chunks = vec![&b"Nushell\xe6"[..], &b"\x9c\x80\xe9"[..], &b"\xab"[..]];
1419 let iter = test_chunks(chunks, ByteStreamType::String);
1420
1421 let mut string = String::new();
1422 for value in iter {
1423 match value {
1424 Ok(value) => string.push_str(&value.into_string().expect("not a string")),
1425 Err(err) => {
1426 println!("string so far: {string:?}");
1427 println!("got error: {err:?}");
1428 assert!(!string.is_empty());
1429 assert!(matches!(err, ShellError::NonUtf8Custom { .. }));
1430 return;
1431 }
1432 }
1433 }
1434 panic!("no error");
1435 }
1436
1437 #[test]
1438 fn chunks_read_unknown_fallback() {
1439 let chunks = vec![&b"Nushell"[..], &b"\x9c\x80\xe9abcd"[..], &b"efgh"[..]];
1440 let mut iter = test_chunks(chunks, ByteStreamType::Unknown);
1441
1442 let mut get = || iter.next().expect("end of iter").expect("error");
1443
1444 assert_eq!(Value::test_string("Nushell"), get());
1445 assert_eq!(Value::test_binary(b"\x9c\x80\xe9abcd"), get());
1446 assert_eq!(Value::test_binary(b"efgh"), get());
1448 }
1449}