Skip to main content

nu_protocol/pipeline/
pipeline_data.rs

1#[cfg(feature = "os")]
2use crate::process::ExitStatusGuard;
3use crate::{
4    ByteStream, ByteStreamSource, ByteStreamType, CompareTypes, Config, ListStream, OutDest,
5    PipelineMetadata, Range, ShellError, Signals, Span, Type, TypeRelation, Value,
6    ast::PathMember,
7    engine::{EngineState, Stack},
8    ir,
9    shell_error::{generic::GenericError, io::IoError},
10};
11use std::{
12    borrow::Cow,
13    io::Write,
14    ops::{Deref, DerefMut},
15    panic::Location,
16};
17
18const LINE_ENDING_PATTERN: &[char] = &['\r', '\n'];
19
20/// The foundational abstraction for input and output to commands
21///
22/// This represents either a single Value or a stream of values coming into the command or leaving a command.
23///
24/// A note on implementation:
25///
26/// We've tried a few variations of this structure. Listing these below so we have a record.
27///
28/// * We tried always assuming a stream in Nushell. This was a great 80% solution, but it had some rough edges.
29///   Namely, how do you know the difference between a single string and a list of one string. How do you know
30///   when to flatten the data given to you from a data source into the stream or to keep it as an unflattened
31///   list?
32///
33/// * We tried putting the stream into Value. This had some interesting properties as now commands "just worked
34///   on values", but lead to a few unfortunate issues.
35///
36/// The first is that you can't easily clone Values in a way that felt largely immutable. For example, if
37/// you cloned a Value which contained a stream, and in one variable drained some part of it, then the second
38/// variable would see different values based on what you did to the first.
39///
40/// To make this kind of mutation thread-safe, we would have had to produce a lock for the stream, which in
41/// practice would have meant always locking the stream before reading from it. But more fundamentally, it
42/// felt wrong in practice that observation of a value at runtime could affect other values which happen to
43/// alias the same stream. By separating these, we don't have this effect. Instead, variables could get
44/// concrete list values rather than streams, and be able to view them without non-local effects.
45///
46/// * A balance of the two approaches is what we've landed on: Values are thread-safe to pass, and we can stream
47///   them into any sources. Streams are still available to model the infinite streams approach of original
48///   Nushell.
49#[derive(Debug)]
50pub enum PipelineData {
51    Empty,
52    Value(Value, Option<PipelineMetadata>),
53    ListStream(ListStream, Option<PipelineMetadata>),
54    ByteStream(ByteStream, Option<PipelineMetadata>),
55}
56
57impl PipelineData {
58    pub const fn empty() -> PipelineData {
59        PipelineData::Empty
60    }
61
62    pub fn value(val: Value, metadata: impl Into<Option<PipelineMetadata>>) -> Self {
63        PipelineData::Value(val, metadata.into())
64    }
65
66    pub fn list_stream(stream: ListStream, metadata: impl Into<Option<PipelineMetadata>>) -> Self {
67        PipelineData::ListStream(stream, metadata.into())
68    }
69
70    pub fn byte_stream(stream: ByteStream, metadata: impl Into<Option<PipelineMetadata>>) -> Self {
71        PipelineData::ByteStream(stream, metadata.into())
72    }
73
74    /// Returns a reference to the metadata if it exists.
75    pub fn metadata_ref(&self) -> Option<&PipelineMetadata> {
76        match self {
77            PipelineData::Empty => None,
78            PipelineData::Value(_, meta)
79            | PipelineData::ListStream(_, meta)
80            | PipelineData::ByteStream(_, meta) => meta.as_ref(),
81        }
82    }
83
84    /// Returns a mutable reference to the metadata if it exists.
85    pub fn metadata_mut(&mut self) -> Option<&mut PipelineMetadata> {
86        match self {
87            PipelineData::Empty => None,
88            PipelineData::Value(_, meta)
89            | PipelineData::ListStream(_, meta)
90            | PipelineData::ByteStream(_, meta) => meta.as_mut(),
91        }
92    }
93
94    /// Take the metadata out of pipeline if it exists.
95    pub fn take_metadata(&mut self) -> Option<PipelineMetadata> {
96        match self {
97            PipelineData::Empty => None,
98            PipelineData::Value(_, meta)
99            | PipelineData::ListStream(_, meta)
100            | PipelineData::ByteStream(_, meta) => meta.take(),
101        }
102    }
103
104    pub fn set_metadata(mut self, metadata: Option<PipelineMetadata>) -> Self {
105        match &mut self {
106            PipelineData::Empty => {}
107            PipelineData::Value(_, meta)
108            | PipelineData::ListStream(_, meta)
109            | PipelineData::ByteStream(_, meta) => *meta = metadata,
110        }
111        self
112    }
113
114    pub fn is_nothing(&self) -> bool {
115        matches!(self, PipelineData::Value(Value::Nothing { .. }, ..))
116            || matches!(self, PipelineData::Empty)
117    }
118
119    /// PipelineData doesn't always have a Span, but we can try!
120    pub fn span(&self) -> Option<Span> {
121        match self {
122            PipelineData::Empty => None,
123            PipelineData::Value(value, ..) => Some(value.span()),
124            PipelineData::ListStream(stream, ..) => Some(stream.span()),
125            PipelineData::ByteStream(stream, ..) => Some(stream.span()),
126        }
127    }
128
129    /// Change the span of the [`PipelineData`].
130    ///
131    /// Returns `Value(Nothing)` with the given span if it was [`PipelineData::empty()`].
132    pub fn with_span(self, span: Span) -> Self {
133        match self {
134            PipelineData::Empty => PipelineData::value(Value::nothing(span), None),
135            PipelineData::Value(value, metadata) => {
136                PipelineData::value(value.with_span(span), metadata)
137            }
138            PipelineData::ListStream(stream, metadata) => {
139                PipelineData::list_stream(stream.with_span(span), metadata)
140            }
141            PipelineData::ByteStream(stream, metadata) => {
142                PipelineData::byte_stream(stream.with_span(span), metadata)
143            }
144        }
145    }
146
147    /// Get a type that is representative of the `PipelineData`.
148    ///
149    /// The type returned here makes no effort to collect a stream, so it may be a different type
150    /// than would be returned by [`Value::get_type()`] on the result of
151    /// [`.into_value()`](Self::into_value).
152    ///
153    /// Specifically, a `ListStream` results in `list<any>` rather than
154    /// the fully complete [`list`](Type::List) type (which would require knowing the contents),
155    /// and a `ByteStream` with [unknown](crate::ByteStreamType::Unknown) type results in
156    /// [`any`](Type::Any) rather than [`string`](Type::String) or [`binary`](Type::Binary).
157    pub fn get_type(&self) -> Type {
158        match self {
159            PipelineData::Empty => Type::Nothing,
160            PipelineData::Value(value, _) => value.get_type(),
161            PipelineData::ListStream(_, _) => Type::list(Type::Any),
162            PipelineData::ByteStream(stream, _) => stream.type_().into(),
163        }
164    }
165
166    pub fn into_value(self, span: Span) -> Result<Value, ShellError> {
167        match self {
168            PipelineData::Empty => Ok(Value::nothing(span)),
169            PipelineData::Value(value, ..) => {
170                if value.span() == Span::unknown() {
171                    Ok(value.with_span(span))
172                } else {
173                    Ok(value)
174                }
175            }
176            PipelineData::ListStream(stream, ..) => stream.into_value(),
177            PipelineData::ByteStream(stream, ..) => stream.into_value(),
178        }
179    }
180
181    /// Converts any `Value` variant that can be represented as a stream into its stream variant.
182    ///
183    /// This means that lists and ranges are converted into list streams, and strings and binary are
184    /// converted into byte streams.
185    ///
186    /// Returns an `Err` with the original stream if the variant couldn't be converted to a stream
187    /// variant. If the variant is already a stream variant, it is returned as-is.
188    pub fn try_into_stream(self, engine_state: &EngineState) -> Result<PipelineData, PipelineData> {
189        let span = self.span().unwrap_or(Span::unknown());
190        match self {
191            PipelineData::ListStream(..) | PipelineData::ByteStream(..) => Ok(self),
192            PipelineData::Value(Value::List { .. } | Value::Range { .. }, ref metadata) => {
193                let metadata = metadata.clone();
194                Ok(PipelineData::list_stream(
195                    ListStream::new(self.into_iter(), span, engine_state.signals().clone()),
196                    metadata,
197                ))
198            }
199            PipelineData::Value(Value::String { val, .. }, metadata) => {
200                Ok(PipelineData::byte_stream(
201                    ByteStream::read_string(val, span, engine_state.signals().clone()),
202                    metadata,
203                ))
204            }
205            PipelineData::Value(Value::Binary { val, .. }, metadata) => {
206                Ok(PipelineData::byte_stream(
207                    ByteStream::read_binary(val, span, engine_state.signals().clone()),
208                    metadata,
209                ))
210            }
211            PipelineData::Value(Value::Custom { val, internal_span }, metadata) => {
212                match val.to_base_value(internal_span) {
213                    Ok(Value::List { vals, .. }) => Ok(PipelineData::list_stream(
214                        ListStream::new(vals.into_iter(), span, engine_state.signals().clone()),
215                        metadata,
216                    )),
217                    Ok(Value::Range { val, .. }) => Ok(PipelineData::list_stream(
218                        ListStream::new(
219                            val.into_range_iter(span, Signals::empty()),
220                            span,
221                            engine_state.signals().clone(),
222                        ),
223                        metadata,
224                    )),
225                    Ok(other) => Err(PipelineData::value(other, metadata)),
226                    Err(_) => Err(PipelineData::Value(
227                        Value::Custom { val, internal_span },
228                        metadata,
229                    )),
230                }
231            }
232            _ => Err(self),
233        }
234    }
235
236    /// Converts this value into a stream when possible, otherwise returns the original value.
237    ///
238    /// This is a convenience wrapper around [`PipelineData::try_into_stream`] for command code
239    /// paths that can operate on both stream and non-stream input without branching.
240    #[must_use]
241    pub fn into_stream_or_original(self, engine_state: &EngineState) -> PipelineData {
242        self.try_into_stream(engine_state)
243            .unwrap_or_else(|original| original)
244    }
245
246    /// Drain and write this [`PipelineData`] to `dest`.
247    ///
248    /// Values are converted to bytes and separated by newlines if this is a `ListStream`.
249    pub fn write_to(self, mut dest: impl Write) -> Result<(), ShellError> {
250        match self {
251            PipelineData::Empty => Ok(()),
252            PipelineData::Value(value, ..) => {
253                let bytes = value_to_bytes(value)?;
254                dest.write_all(&bytes).map_err(|err| {
255                    IoError::new_internal(err, "Could not write PipelineData to dest")
256                })?;
257                dest.flush().map_err(|err| {
258                    IoError::new_internal(err, "Could not flush PipelineData to dest")
259                })?;
260                Ok(())
261            }
262            PipelineData::ListStream(stream, ..) => {
263                for value in stream {
264                    let bytes = value_to_bytes(value)?;
265                    dest.write_all(&bytes).map_err(|err| {
266                        IoError::new_internal(err, "Could not write PipelineData to dest")
267                    })?;
268                    dest.write_all(b"\n").map_err(|err| {
269                        IoError::new_internal(
270                            err,
271                            "Could not write linebreak after PipelineData to dest",
272                        )
273                    })?;
274                }
275                dest.flush().map_err(|err| {
276                    IoError::new_internal(err, "Could not flush PipelineData to dest")
277                })?;
278                Ok(())
279            }
280            PipelineData::ByteStream(stream, ..) => stream.write_to(dest),
281        }
282    }
283
284    /// Write the data to `dest` like [`write_to`](Self::write_to), for an `o>` redirection, and
285    /// return what is left for the next command.
286    ///
287    /// That's nothing, except for an external whose stderr has its own pipe, meaning the next
288    /// command reads stdout and stderr apart: only its stdout is written, and it comes back with
289    /// its stderr (see [`ByteStream::write_stdout_to`]).
290    pub fn write_stdout_to(self, dest: impl Write) -> Result<Self, ShellError> {
291        match self {
292            #[cfg(feature = "os")]
293            PipelineData::ByteStream(stream, metadata) => Ok(stream
294                .write_stdout_to(dest)?
295                .map_or(PipelineData::Empty, |stream| {
296                    PipelineData::byte_stream(stream, metadata)
297                })),
298            data => data.write_to(dest).map(|()| PipelineData::Empty),
299        }
300    }
301
302    /// Drain this [`PipelineData`] according to the current stdout [`OutDest`]s in `stack`.
303    ///
304    /// For [`OutDest::Pipe`], [`OutDest::PipeRaw`] and [`OutDest::PipeSeparate`], this will return
305    /// the [`PipelineData`] as is. For [`OutDest::Value`], this will collect into a value and return it. For
306    /// [`OutDest::Print`], the [`PipelineData`] is drained and printed. Otherwise, the
307    /// [`PipelineData`] is drained, but only printed if it is the output of an external command.
308    pub fn drain_to_out_dests(
309        mut self,
310        engine_state: &EngineState,
311        stack: &mut Stack,
312    ) -> Result<Self, ShellError> {
313        match stack.pipe_stdout().unwrap_or(&OutDest::Inherit) {
314            OutDest::Print => {
315                self.print_table(engine_state, stack, false, false)?;
316                Ok(Self::Empty)
317            }
318            OutDest::Pipe | OutDest::PipeRaw | OutDest::PipeSeparate => Ok(self),
319            OutDest::Value => {
320                let metadata = self.take_metadata();
321                let span = self.span().unwrap_or(Span::unknown());
322                self.into_value(span).map(|val| Self::Value(val, metadata))
323            }
324            OutDest::File(file) => self.write_stdout_to(file.as_ref()),
325            OutDest::Null | OutDest::Inherit => {
326                self.drain()?;
327                Ok(Self::Empty)
328            }
329        }
330    }
331
332    pub fn drain(self) -> Result<(), ShellError> {
333        match self {
334            Self::Empty => Ok(()),
335            Self::Value(Value::Error { error, .. }, ..) => Err(*error),
336            Self::Value(..) => Ok(()),
337            Self::ListStream(stream, ..) => stream.drain(),
338            Self::ByteStream(stream, ..) => stream.drain(),
339        }
340    }
341
342    /// Try convert from self into iterator
343    ///
344    /// It returns Err if the `self` cannot be converted to an iterator.
345    ///
346    /// The `span` should be the span of the command or operation that would raise an error.
347    pub fn into_iter_strict(self, span: Span) -> Result<PipelineIterator, ShellError> {
348        Ok(PipelineIterator(match self {
349            PipelineData::Value(value, ..) => {
350                let val_span = value.span();
351                match value {
352                    Value::List { vals, .. } => PipelineIteratorInner::ListStream(
353                        ListStream::new(vals.into_iter(), val_span, Signals::empty()).into_iter(),
354                    ),
355                    Value::Binary { val, .. } => PipelineIteratorInner::ListStream(
356                        ListStream::new(
357                            val.into_owned()
358                                .into_iter()
359                                .map(move |x| Value::int(x as i64, val_span)),
360                            val_span,
361                            Signals::empty(),
362                        )
363                        .into_iter(),
364                    ),
365                    Value::Range { val, signals, .. } => PipelineIteratorInner::ListStream(
366                        ListStream::new(
367                            val.into_range_iter(val_span, signals.unwrap_or_else(Signals::empty)),
368                            val_span,
369                            Signals::empty(),
370                        )
371                        .into_iter(),
372                    ),
373                    // Handle iterable custom values by converting to base value first
374                    Value::Custom { ref val, .. } if val.is_iterable() => {
375                        match val.to_base_value(val_span) {
376                            Ok(Value::List { vals, .. }) => PipelineIteratorInner::ListStream(
377                                ListStream::new(vals.into_iter(), val_span, Signals::empty())
378                                    .into_iter(),
379                            ),
380                            Ok(other) => {
381                                return Err(ShellError::OnlySupportsThisInputType {
382                                    exp_input_type: "list, binary, range, or byte stream".into(),
383                                    wrong_type: other.get_type().to_string(),
384                                    dst_span: span,
385                                    src_span: val_span,
386                                });
387                            }
388                            Err(err) => return Err(err),
389                        }
390                    }
391                    // Propagate errors by explicitly matching them before the final case.
392                    Value::Error { error, .. } => return Err(*error),
393                    other => {
394                        return Err(ShellError::OnlySupportsThisInputType {
395                            exp_input_type: "list, binary, range, or byte stream".into(),
396                            wrong_type: other.get_type().to_string(),
397                            dst_span: span,
398                            src_span: val_span,
399                        });
400                    }
401                }
402            }
403            PipelineData::ListStream(stream, ..) => {
404                PipelineIteratorInner::ListStream(stream.into_iter())
405            }
406            PipelineData::Empty => {
407                return Err(ShellError::OnlySupportsThisInputType {
408                    exp_input_type: "list, binary, range, or byte stream".into(),
409                    wrong_type: "null".into(),
410                    dst_span: span,
411                    src_span: span,
412                });
413            }
414            PipelineData::ByteStream(stream, ..) => {
415                if let Some(chunks) = stream.chunks() {
416                    PipelineIteratorInner::ByteStream(chunks)
417                } else {
418                    PipelineIteratorInner::Empty
419                }
420            }
421        }))
422    }
423
424    pub fn collect_string(self, separator: &str, config: &Config) -> Result<String, ShellError> {
425        match self {
426            PipelineData::Empty => Ok(String::new()),
427            PipelineData::Value(value, ..) => Ok(value.to_expanded_string(separator, config)),
428            PipelineData::ListStream(stream, ..) => Ok(stream.into_string(separator, config)),
429            PipelineData::ByteStream(stream, ..) => stream.into_string(),
430        }
431    }
432
433    /// Retrieves string from pipeline data.
434    ///
435    /// As opposed to `collect_string` this raises error rather than converting non-string values.
436    /// The `span` will be used if `ListStream` is encountered since it doesn't carry a span.
437    pub fn collect_string_strict(
438        self,
439        span: Span,
440    ) -> Result<(String, Span, Option<PipelineMetadata>), ShellError> {
441        match self {
442            PipelineData::Empty => Ok((String::new(), span, None)),
443            PipelineData::Value(Value::String { val, .. }, metadata) => Ok((val, span, metadata)),
444            PipelineData::Value(val, ..) => Err(ShellError::TypeMismatch {
445                err_message: "string".into(),
446                span: val.span(),
447            }),
448            PipelineData::ListStream(..) => Err(ShellError::TypeMismatch {
449                err_message: "string".into(),
450                span,
451            }),
452            PipelineData::ByteStream(stream, metadata) => {
453                let span = stream.span();
454                Ok((stream.into_string()?, span, metadata))
455            }
456        }
457    }
458
459    pub fn follow_cell_path(
460        self,
461        cell_path: &[PathMember],
462        head: Span,
463    ) -> Result<Value, ShellError> {
464        match self {
465            // FIXME: there are probably better ways of doing this
466            PipelineData::ListStream(stream, ..) => Value::list(stream.into_iter().collect(), head)
467                .follow_cell_path(cell_path)
468                .map(Cow::into_owned),
469            PipelineData::Value(v, ..) => v.follow_cell_path(cell_path).map(Cow::into_owned),
470            PipelineData::Empty => Err(ShellError::IncompatiblePathAccess {
471                type_name: "empty pipeline".to_string(),
472                span: head,
473            }),
474            PipelineData::ByteStream(stream, ..) => Err(ShellError::IncompatiblePathAccess {
475                type_name: stream.type_().describe().to_owned(),
476                span: stream.span(),
477            }),
478        }
479    }
480
481    /// Simplified mapper to help with simple values also. For full iterator support use `.into_iter()` instead
482    pub fn map<F>(self, mut f: F, signals: &Signals) -> Result<PipelineData, ShellError>
483    where
484        Self: Sized,
485        F: FnMut(Value) -> Value + 'static + Send,
486    {
487        match self {
488            PipelineData::Value(value, metadata) => {
489                let span = value.span();
490                let pipeline = match value {
491                    Value::List { vals, .. } => vals
492                        .into_iter()
493                        .map(f)
494                        .into_pipeline_data(span, signals.clone()),
495                    Value::Range { val, .. } => val
496                        .into_range_iter(span, Signals::empty())
497                        .map(f)
498                        .into_pipeline_data(span, signals.clone()),
499                    Value::Custom { ref val, .. } if val.is_iterable() => {
500                        match val.to_base_value(span)? {
501                            Value::List { vals, .. } => vals
502                                .into_iter()
503                                .map(f)
504                                .into_pipeline_data(span, signals.clone()),
505                            Value::Range { val, .. } => val
506                                .into_range_iter(span, Signals::empty())
507                                .map(f)
508                                .into_pipeline_data(span, signals.clone()),
509                            value => match f(value) {
510                                Value::Error { error, .. } => return Err(*error),
511                                v => v.into_pipeline_data(),
512                            },
513                        }
514                    }
515                    value => match f(value) {
516                        Value::Error { error, .. } => return Err(*error),
517                        v => v.into_pipeline_data(),
518                    },
519                };
520                Ok(pipeline.set_metadata(metadata))
521            }
522            PipelineData::Empty => Ok(PipelineData::empty()),
523            PipelineData::ListStream(stream, metadata) => {
524                Ok(PipelineData::list_stream(stream.map(f), metadata))
525            }
526            PipelineData::ByteStream(stream, metadata) => {
527                Ok(f(stream.into_value()?).into_pipeline_data_with_metadata(metadata))
528            }
529        }
530    }
531
532    /// Simplified flatmapper. For full iterator support use `.into_iter()` instead
533    pub fn flat_map<U, F>(self, mut f: F, signals: &Signals) -> Result<PipelineData, ShellError>
534    where
535        Self: Sized,
536        U: IntoIterator<Item = Value> + 'static,
537        <U as IntoIterator>::IntoIter: 'static + Send,
538        F: FnMut(Value) -> U + 'static + Send,
539    {
540        match self {
541            PipelineData::Empty => Ok(PipelineData::empty()),
542            PipelineData::Value(value, metadata) => {
543                let span = value.span();
544                let pipeline = match value {
545                    Value::List { vals, .. } => vals
546                        .into_iter()
547                        .flat_map(f)
548                        .into_pipeline_data(span, signals.clone()),
549                    Value::Range { val, .. } => val
550                        .into_range_iter(span, Signals::empty())
551                        .flat_map(f)
552                        .into_pipeline_data(span, signals.clone()),
553                    Value::Custom { ref val, .. } if val.is_iterable() => {
554                        match val.to_base_value(span)? {
555                            Value::List { vals, .. } => vals
556                                .into_iter()
557                                .flat_map(f)
558                                .into_pipeline_data(span, signals.clone()),
559                            Value::Range { val, .. } => val
560                                .into_range_iter(span, Signals::empty())
561                                .flat_map(f)
562                                .into_pipeline_data(span, signals.clone()),
563                            value => f(value)
564                                .into_iter()
565                                .into_pipeline_data(span, signals.clone()),
566                        }
567                    }
568                    value => f(value)
569                        .into_iter()
570                        .into_pipeline_data(span, signals.clone()),
571                };
572                Ok(pipeline.set_metadata(metadata))
573            }
574            PipelineData::ListStream(stream, metadata) => Ok(PipelineData::list_stream(
575                stream.modify(|iter| iter.flat_map(f)),
576                metadata,
577            )),
578            PipelineData::ByteStream(stream, metadata) => {
579                // TODO: is this behavior desired / correct ?
580                let span = stream.span();
581                let iter = match String::from_utf8(stream.into_bytes()?) {
582                    Ok(mut str) => {
583                        str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len());
584                        f(Value::string(str, span))
585                    }
586                    Err(err) => f(Value::binary(err.into_bytes(), span)),
587                };
588                Ok(iter.into_iter().into_pipeline_data_with_metadata(
589                    span,
590                    signals.clone(),
591                    metadata,
592                ))
593            }
594        }
595    }
596
597    pub fn filter<F>(self, mut f: F, signals: &Signals) -> Result<PipelineData, ShellError>
598    where
599        Self: Sized,
600        F: FnMut(&Value) -> bool + 'static + Send,
601    {
602        match self {
603            PipelineData::Empty => Ok(PipelineData::empty()),
604            PipelineData::Value(value, metadata) => {
605                let span = value.span();
606                let pipeline = match value {
607                    Value::List { vals, .. } => vals
608                        .into_iter()
609                        .filter(f)
610                        .into_pipeline_data(span, signals.clone()),
611                    Value::Range { val, .. } => val
612                        .into_range_iter(span, Signals::empty())
613                        .filter(f)
614                        .into_pipeline_data(span, signals.clone()),
615                    Value::Custom { ref val, .. } if val.is_iterable() => {
616                        match val.to_base_value(span)? {
617                            Value::List { vals, .. } => vals
618                                .into_iter()
619                                .filter(f)
620                                .into_pipeline_data(span, signals.clone()),
621                            Value::Range { val, .. } => val
622                                .into_range_iter(span, Signals::empty())
623                                .filter(f)
624                                .into_pipeline_data(span, signals.clone()),
625                            value => {
626                                if f(&value) {
627                                    value.into_pipeline_data()
628                                } else {
629                                    Value::nothing(span).into_pipeline_data()
630                                }
631                            }
632                        }
633                    }
634                    value => {
635                        if f(&value) {
636                            value.into_pipeline_data()
637                        } else {
638                            Value::nothing(span).into_pipeline_data()
639                        }
640                    }
641                };
642                Ok(pipeline.set_metadata(metadata))
643            }
644            PipelineData::ListStream(stream, metadata) => Ok(PipelineData::list_stream(
645                stream.modify(|iter| iter.filter(f)),
646                metadata,
647            )),
648            PipelineData::ByteStream(stream, metadata) => {
649                // TODO: is this behavior desired / correct ?
650                let span = stream.span();
651                let value = match String::from_utf8(stream.into_bytes()?) {
652                    Ok(mut str) => {
653                        str.truncate(str.trim_end_matches(LINE_ENDING_PATTERN).len());
654                        Value::string(str, span)
655                    }
656                    Err(err) => Value::binary(err.into_bytes(), span),
657                };
658                let value = if f(&value) {
659                    value
660                } else {
661                    Value::nothing(span)
662                };
663                Ok(value.into_pipeline_data_with_metadata(metadata))
664            }
665        }
666    }
667
668    /// Try to convert Value from Value::Range to Value::List.
669    /// This is useful to expand Value::Range into array notation, specifically when
670    /// converting `to json` or `to nuon`.
671    /// `1..3 | to XX -> [1,2,3]`
672    pub fn try_expand_range(self) -> Result<PipelineData, ShellError> {
673        match self {
674            PipelineData::Value(v, metadata) => {
675                let span = v.span();
676                match v {
677                    Value::Range { val, signals, .. } => {
678                        match *val {
679                            Range::IntRange(range) => {
680                                if range.is_unbounded() {
681                                    return Err(ShellError::Generic(
682                                        GenericError::new(
683                                            "Cannot create range",
684                                            "Unbounded ranges are not allowed when converting to this format",
685                                            span,
686                                        )
687                                        .with_help(
688                                            "Consider using ranges with valid start and end point.",
689                                        ),
690                                    ));
691                                }
692                            }
693                            Range::FloatRange(range) => {
694                                if range.is_unbounded() {
695                                    return Err(ShellError::Generic(
696                                        GenericError::new(
697                                            "Cannot create range",
698                                            "Unbounded ranges are not allowed when converting to this format",
699                                            span,
700                                        )
701                                        .with_help(
702                                            "Consider using ranges with valid start and end point.",
703                                        ),
704                                    ));
705                                }
706                            }
707                        }
708                        let range_values: Vec<Value> = val
709                            .into_range_iter(span, signals.unwrap_or_else(Signals::empty))
710                            .collect();
711                        Ok(PipelineData::value(Value::list(range_values, span), None))
712                    }
713                    x => Ok(PipelineData::value(x, metadata)),
714                }
715            }
716            _ => Ok(self),
717        }
718    }
719
720    /// Consume and print self data immediately, formatted using table command.
721    ///
722    /// This does not respect the display_output hook. If a value is being printed out by a command,
723    /// this function should be used. Otherwise, `nu_cli::util::print_pipeline` should be preferred.
724    ///
725    /// `no_newline` controls if we need to attach newline character to output.
726    /// `to_stderr` controls if data is output to stderr, when the value is false, the data is output to stdout.
727    pub fn print_table(
728        self,
729        engine_state: &EngineState,
730        stack: &mut Stack,
731        no_newline: bool,
732        to_stderr: bool,
733    ) -> Result<(), ShellError> {
734        match self {
735            // Print byte streams directly as long as they aren't binary.
736            PipelineData::ByteStream(stream, ..) if stream.type_() != ByteStreamType::Binary => {
737                stream.print(to_stderr)
738            }
739            _ => {
740                // If the table function is in the declarations, then we can use it
741                // to create the table value that will be printed in the terminal
742                if let Some(decl_id) = engine_state.table_decl_id {
743                    let command = engine_state.get_decl(decl_id);
744                    if command.block_id().is_some() {
745                        self.write_all_and_flush(engine_state, no_newline, to_stderr)
746                    } else {
747                        let call = ir::Call::build(decl_id, Span::new(0, 0)).finish();
748                        let table = command.run(engine_state, stack, &(&call).into(), self)?;
749                        table.write_all_and_flush(engine_state, no_newline, to_stderr)
750                    }
751                } else {
752                    self.write_all_and_flush(engine_state, no_newline, to_stderr)
753                }
754            }
755        }
756    }
757
758    /// Consume and print self data without any extra formatting.
759    ///
760    /// This does not use the `table` command to format data, and also prints binary values and
761    /// streams in their raw format without generating a hexdump first.
762    ///
763    /// `no_newline` controls if we need to attach newline character to output.
764    /// `to_stderr` controls if data is output to stderr, when the value is false, the data is output to stdout.
765    pub fn print_raw(
766        self,
767        engine_state: &EngineState,
768        no_newline: bool,
769        to_stderr: bool,
770    ) -> Result<(), ShellError> {
771        let span = self.span();
772        if let PipelineData::Value(Value::Binary { val: bytes, .. }, _) = self {
773            if to_stderr {
774                write_all_and_flush(
775                    bytes.as_slice(),
776                    &mut std::io::stderr().lock(),
777                    "stderr",
778                    span,
779                    engine_state.signals(),
780                )?;
781            } else {
782                write_all_and_flush(
783                    bytes.as_slice(),
784                    &mut std::io::stdout().lock(),
785                    "stdout",
786                    span,
787                    engine_state.signals(),
788                )?;
789            }
790            Ok(())
791        } else {
792            self.write_all_and_flush(engine_state, no_newline, to_stderr)
793        }
794    }
795
796    fn write_all_and_flush(
797        self,
798        engine_state: &EngineState,
799        no_newline: bool,
800        to_stderr: bool,
801    ) -> Result<(), ShellError> {
802        let span = self.span();
803        if let PipelineData::ByteStream(stream, ..) = self {
804            // Copy ByteStreams directly
805            stream.print(to_stderr)
806        } else {
807            let config = engine_state.get_config();
808            for item in self {
809                let mut out = if let Value::Error { error, .. } = item {
810                    return Err(*error);
811                } else {
812                    item.to_expanded_string("\n", config)
813                };
814
815                if !no_newline {
816                    out.push('\n');
817                }
818
819                if to_stderr {
820                    write_all_and_flush(
821                        out,
822                        &mut std::io::stderr().lock(),
823                        "stderr",
824                        span,
825                        engine_state.signals(),
826                    )?;
827                } else {
828                    write_all_and_flush(
829                        out,
830                        &mut std::io::stdout().lock(),
831                        "stdout",
832                        span,
833                        engine_state.signals(),
834                    )?;
835                }
836            }
837
838            Ok(())
839        }
840    }
841
842    pub fn unsupported_input_error(
843        self,
844        expected_type: impl Into<String>,
845        span: Span,
846    ) -> ShellError {
847        match self {
848            PipelineData::Empty => ShellError::PipelineEmpty { dst_span: span },
849            PipelineData::Value(value, ..) => ShellError::OnlySupportsThisInputType {
850                exp_input_type: expected_type.into(),
851                wrong_type: value.get_type().get_non_specified_string(),
852                dst_span: span,
853                src_span: value.span(),
854            },
855            PipelineData::ListStream(stream, ..) => ShellError::OnlySupportsThisInputType {
856                exp_input_type: expected_type.into(),
857                wrong_type: "list (stream)".into(),
858                dst_span: span,
859                src_span: stream.span(),
860            },
861            PipelineData::ByteStream(stream, ..) => ShellError::OnlySupportsThisInputType {
862                exp_input_type: expected_type.into(),
863                wrong_type: stream.type_().describe().into(),
864                dst_span: span,
865                src_span: stream.span(),
866            },
867        }
868    }
869
870    // PipelineData might connect to a running process which has an exit status future
871    // Use this method to retrieve that future, it's useful for implementing `pipefail` feature.
872    #[cfg(feature = "os")]
873    pub fn clone_exit_status_future(&self) -> Option<ExitStatusGuard> {
874        match self {
875            PipelineData::Empty | PipelineData::Value(..) | PipelineData::ListStream(..) => None,
876            PipelineData::ByteStream(stream, ..) => match stream.source() {
877                ByteStreamSource::Read(..) | ByteStreamSource::File(..) => None,
878                ByteStreamSource::Child(c) => {
879                    let exit_future = c.clone_exit_status_future();
880                    let ignore_error = c.clone_ignore_error();
881                    Some(ExitStatusGuard::new(exit_future, ignore_error))
882                }
883            },
884        }
885    }
886}
887
888impl CompareTypes<Type> for PipelineData {
889    fn compare_types(&self, other: &Type) -> Option<TypeRelation> {
890        let self_ty = match self {
891            PipelineData::Empty => Type::Nothing,
892            PipelineData::ListStream(_, _) => Type::list(Type::Any),
893            PipelineData::ByteStream(stream, _) => stream.type_().into(),
894            PipelineData::Value(value, _) => return value.compare_types(other),
895        };
896        self_ty.compare_types(other)
897    }
898
899    /// Determine if the `PipelineData` can be assigned to `other`.
900    ///
901    /// This check makes no effort to collect a stream, so it may be a different result
902    /// than would be returned by calling [`Value::is_subtype_of()`] on the result of
903    /// [`.into_value()`](Self::into_value).
904    ///
905    /// A `ListStream` acts the same as an empty list type: it is a subtype of any [`list`](Type::List)
906    /// or [`table`](Type::Table) type. After converting to a value, it may become a more specific type.
907    /// For example, a `ListStream` is a subtype of `list<int>` and `list<string>`.
908    /// If calling [`.into_value()`](Self::into_value) results in a `list<int>`,
909    /// then the value would not be a subtype of `list<string>`, in contrast to the original `ListStream`.
910    ///
911    /// A `ByteStream` is a subtype of [`string`](Type::String) if it is coercible into a string.
912    /// Likewise, a `ByteStream` is a subtype of [`binary`](Type::Binary) if it is coercible into a binary value.
913    fn is_assignable_to(&self, dst: &Type) -> bool {
914        let self_ty = match self {
915            PipelineData::Empty => Type::Nothing,
916            PipelineData::ListStream(_, _) => Type::list(Type::Any),
917            PipelineData::ByteStream(stream, _) => stream.type_().into(),
918            PipelineData::Value(value, _) => return value.is_assignable_to(dst),
919        };
920        self_ty.is_assignable_to(dst)
921    }
922}
923
924pub fn write_all_and_flush<T>(
925    data: T,
926    destination: &mut impl Write,
927    destination_name: &str,
928    span: Option<Span>,
929    signals: &Signals,
930) -> Result<(), ShellError>
931where
932    T: AsRef<[u8]>,
933{
934    let io_error_map = |err: std::io::Error, location: &Location<'_>| {
935        let context = format!("Writing to {destination_name} failed");
936        match span {
937            None => IoError::new_internal_with_location(err, context, location),
938            Some(span) if span == Span::unknown() => {
939                IoError::new_internal_with_location(err, context, location)
940            }
941            Some(span) => IoError::new_with_additional_context(err, span, None, context),
942        }
943    };
944
945    let span = span.unwrap_or(Span::unknown());
946    const OUTPUT_CHUNK_SIZE: usize = 8192;
947    for chunk in data.as_ref().chunks(OUTPUT_CHUNK_SIZE) {
948        signals.check(&span)?;
949        destination
950            .write_all(chunk)
951            .map_err(|err| io_error_map(err, Location::caller()))?;
952    }
953    destination
954        .flush()
955        .map_err(|err| io_error_map(err, Location::caller()))?;
956    Ok(())
957}
958
959/// Write collected bytes to the requested output destination.
960///
961/// A pipe, value, or inherited destination has nothing to write the bytes into, so they go to
962/// the process's stdout when `stdout_fallback` is set, and to its stderr otherwise.
963pub fn write_to_out_dest(
964    bytes: &[u8],
965    out_dest: &OutDest,
966    span: Span,
967    stdout_fallback: bool,
968    signals: &Signals,
969) -> Result<(), ShellError> {
970    if bytes.is_empty() {
971        return Ok(());
972    }
973
974    match out_dest {
975        OutDest::Null => Ok(()),
976        OutDest::File(file) => {
977            let mut file = file.as_ref();
978            write_all_and_flush(bytes, &mut file, "file", Some(span), signals)
979        }
980        OutDest::Print
981        | OutDest::Inherit
982        | OutDest::Pipe
983        | OutDest::PipeRaw
984        | OutDest::PipeSeparate
985        | OutDest::Value => {
986            if stdout_fallback {
987                let mut stdout = std::io::stdout();
988                write_all_and_flush(bytes, &mut stdout, "stdout", Some(span), signals)
989            } else {
990                let mut stderr = std::io::stderr();
991                write_all_and_flush(bytes, &mut stderr, "stderr", Some(span), signals)
992            }
993        }
994    }
995}
996
997enum PipelineIteratorInner {
998    Empty,
999    Value(Value),
1000    ListStream(crate::list_stream::IntoIter),
1001    ByteStream(crate::byte_stream::Chunks),
1002}
1003
1004pub struct PipelineIterator(PipelineIteratorInner);
1005
1006impl IntoIterator for PipelineData {
1007    type Item = Value;
1008
1009    type IntoIter = PipelineIterator;
1010
1011    fn into_iter(self) -> Self::IntoIter {
1012        PipelineIterator(match self {
1013            PipelineData::Empty => PipelineIteratorInner::Empty,
1014            PipelineData::Value(value, ..) => {
1015                let span = value.span();
1016                match value {
1017                    Value::List { vals, signals, .. } => PipelineIteratorInner::ListStream(
1018                        ListStream::new(
1019                            vals.into_iter(),
1020                            span,
1021                            signals.unwrap_or_else(Signals::empty),
1022                        )
1023                        .into_iter(),
1024                    ),
1025                    Value::Range { val, signals, .. } => PipelineIteratorInner::ListStream(
1026                        ListStream::new(
1027                            val.into_range_iter(span, signals.unwrap_or_else(Signals::empty)),
1028                            span,
1029                            Signals::empty(),
1030                        )
1031                        .into_iter(),
1032                    ),
1033                    // Handle iterable custom values by converting to base value first
1034                    Value::Custom { ref val, .. } if val.is_iterable() => {
1035                        match val.to_base_value(span) {
1036                            Ok(Value::List { vals, signals, .. }) => {
1037                                PipelineIteratorInner::ListStream(
1038                                    ListStream::new(
1039                                        vals.into_iter(),
1040                                        span,
1041                                        signals.unwrap_or_else(Signals::empty),
1042                                    )
1043                                    .into_iter(),
1044                                )
1045                            }
1046                            Ok(other) => PipelineIteratorInner::Value(other),
1047                            Err(err) => PipelineIteratorInner::Value(Value::error(err, span)),
1048                        }
1049                    }
1050                    x => PipelineIteratorInner::Value(x),
1051                }
1052            }
1053            PipelineData::ListStream(stream, ..) => {
1054                PipelineIteratorInner::ListStream(stream.into_iter())
1055            }
1056            PipelineData::ByteStream(stream, ..) => stream.chunks().map_or(
1057                PipelineIteratorInner::Empty,
1058                PipelineIteratorInner::ByteStream,
1059            ),
1060        })
1061    }
1062}
1063
1064impl Iterator for PipelineIterator {
1065    type Item = Value;
1066
1067    fn next(&mut self) -> Option<Self::Item> {
1068        match &mut self.0 {
1069            PipelineIteratorInner::Empty => None,
1070            PipelineIteratorInner::Value(Value::Nothing { .. }, ..) => None,
1071            PipelineIteratorInner::Value(v, ..) => Some(std::mem::take(v)),
1072            PipelineIteratorInner::ListStream(stream, ..) => stream.next(),
1073            PipelineIteratorInner::ByteStream(stream) => stream.next().map(|x| match x {
1074                Ok(x) => x,
1075                Err(err) => Value::error(
1076                    err,
1077                    Span::unknown(), //FIXME: unclear where this span should come from
1078                ),
1079            }),
1080        }
1081    }
1082}
1083
1084pub trait IntoPipelineData {
1085    fn into_pipeline_data(self) -> PipelineData;
1086
1087    fn into_pipeline_data_with_metadata(
1088        self,
1089        metadata: impl Into<Option<PipelineMetadata>>,
1090    ) -> PipelineData;
1091}
1092
1093impl<V> IntoPipelineData for V
1094where
1095    V: Into<Value>,
1096{
1097    fn into_pipeline_data(self) -> PipelineData {
1098        PipelineData::value(self.into(), None)
1099    }
1100
1101    fn into_pipeline_data_with_metadata(
1102        self,
1103        metadata: impl Into<Option<PipelineMetadata>>,
1104    ) -> PipelineData {
1105        PipelineData::value(self.into(), metadata.into())
1106    }
1107}
1108
1109pub trait IntoInterruptiblePipelineData {
1110    fn into_pipeline_data(self, span: Span, signals: Signals) -> PipelineData;
1111    fn into_pipeline_data_with_metadata(
1112        self,
1113        span: Span,
1114        signals: Signals,
1115        metadata: impl Into<Option<PipelineMetadata>>,
1116    ) -> PipelineData;
1117}
1118
1119impl<I> IntoInterruptiblePipelineData for I
1120where
1121    I: IntoIterator + Send + 'static,
1122    I::IntoIter: Send + 'static,
1123    <I::IntoIter as Iterator>::Item: Into<Value>,
1124{
1125    fn into_pipeline_data(self, span: Span, signals: Signals) -> PipelineData {
1126        ListStream::new(self.into_iter().map(Into::into), span, signals).into()
1127    }
1128
1129    fn into_pipeline_data_with_metadata(
1130        self,
1131        span: Span,
1132        signals: Signals,
1133        metadata: impl Into<Option<PipelineMetadata>>,
1134    ) -> PipelineData {
1135        PipelineData::list_stream(
1136            ListStream::new(self.into_iter().map(Into::into), span, signals),
1137            metadata.into(),
1138        )
1139    }
1140}
1141
1142fn value_to_bytes(value: Value) -> Result<Vec<u8>, ShellError> {
1143    let bytes = match value {
1144        Value::String { val, .. } => val.into_bytes(),
1145        Value::Binary { val, .. } => val.into_owned(),
1146        Value::List { vals, .. } => {
1147            let val = vals
1148                .into_iter()
1149                .map(Value::coerce_into_string)
1150                .collect::<Result<Vec<String>, ShellError>>()?
1151                .join("\n")
1152                + "\n";
1153
1154            val.into_bytes()
1155        }
1156        // Propagate errors by explicitly matching them before the final case.
1157        Value::Error { error, .. } => return Err(*error),
1158        value => value.coerce_into_string()?.into_bytes(),
1159    };
1160    Ok(bytes)
1161}
1162
1163/// A wrapper to [`PipelineData`] which can also track exit status.
1164///
1165/// We use exit status tracking to implement the `pipefail` feature.
1166#[derive(Debug)]
1167pub struct PipelineExecutionData {
1168    pub body: PipelineData,
1169    #[cfg(feature = "os")]
1170    pub exit: Vec<Option<ExitStatusGuard>>,
1171    /// Whether this data was produced by an early `return` from the block, rather than by
1172    /// evaluating to the end of the block.
1173    ///
1174    /// The flag exists for a single consumer: top-level file evaluation reads it to detect a
1175    /// top-level `return` in a script and skip running `main`. Custom command calls and closure
1176    /// invocations instead clear it via
1177    /// [`eval_block_with_early_return`](https://docs.rs/nu-engine/latest/nu_engine/fn.eval_block_with_early_return.html),
1178    /// so it never leaks past a nested call and only ever reflects a `return` at the current level.
1179    pub early_return: bool,
1180}
1181
1182impl Deref for PipelineExecutionData {
1183    type Target = PipelineData;
1184
1185    fn deref(&self) -> &Self::Target {
1186        &self.body
1187    }
1188}
1189
1190impl DerefMut for PipelineExecutionData {
1191    fn deref_mut(&mut self) -> &mut Self::Target {
1192        &mut self.body
1193    }
1194}
1195
1196impl PipelineExecutionData {
1197    pub fn empty() -> Self {
1198        Self {
1199            body: PipelineData::empty(),
1200            #[cfg(feature = "os")]
1201            exit: vec![],
1202            early_return: false,
1203        }
1204    }
1205
1206    /// Mark this data as having been produced by an early `return`.
1207    pub fn with_early_return(mut self) -> Self {
1208        self.early_return = true;
1209        self
1210    }
1211}
1212
1213impl From<PipelineData> for PipelineExecutionData {
1214    #[cfg(feature = "os")]
1215    fn from(value: PipelineData) -> Self {
1216        let value_span = value.span().unwrap_or_else(Span::unknown);
1217        let exit_status_future = value
1218            .clone_exit_status_future()
1219            .map(|f| f.with_span(value_span));
1220        Self {
1221            body: value,
1222            exit: vec![exit_status_future],
1223            early_return: false,
1224        }
1225    }
1226
1227    #[cfg(not(feature = "os"))]
1228    fn from(value: PipelineData) -> Self {
1229        Self {
1230            body: value,
1231            early_return: false,
1232        }
1233    }
1234}