polars_core/series/
from.rs

1#[cfg(feature = "dtype-categorical")]
2use arrow::compute::concatenate::concatenate_unchecked;
3use arrow::datatypes::Metadata;
4#[cfg(any(
5    feature = "dtype-date",
6    feature = "dtype-datetime",
7    feature = "dtype-time",
8    feature = "dtype-duration"
9))]
10use arrow::temporal_conversions::*;
11use polars_compute::cast::cast_unchecked as cast;
12use polars_error::feature_gated;
13use polars_utils::itertools::Itertools;
14
15use crate::chunked_array::cast::{CastOptions, cast_chunks};
16#[cfg(feature = "object")]
17use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
18#[cfg(feature = "object")]
19use crate::chunked_array::object::registry::get_object_builder;
20use crate::prelude::*;
21
22impl Series {
23    pub fn from_chunk_and_dtype(
24        name: PlSmallStr,
25        chunk: ArrayRef,
26        dtype: &DataType,
27    ) -> PolarsResult<Self> {
28        if &dtype.to_physical().to_arrow(CompatLevel::newest()) != chunk.dtype() {
29            polars_bail!(
30                InvalidOperation: "cannot create a series of type '{dtype}' of arrow chunk with type '{:?}'",
31                chunk.dtype()
32            );
33        }
34
35        // SAFETY: We check that the datatype matches.
36        let series = unsafe { Self::from_chunks_and_dtype_unchecked(name, vec![chunk], dtype) };
37        Ok(series)
38    }
39
40    /// Takes chunks and a polars datatype and constructs the Series
41    /// This is faster than creating from chunks and an arrow datatype because there is no
42    /// casting involved
43    ///
44    /// # Safety
45    ///
46    /// The caller must ensure that the given `dtype`'s physical type matches all the `ArrayRef` dtypes.
47    pub unsafe fn from_chunks_and_dtype_unchecked(
48        name: PlSmallStr,
49        chunks: Vec<ArrayRef>,
50        dtype: &DataType,
51    ) -> Self {
52        use DataType::*;
53        match dtype {
54            #[cfg(feature = "dtype-i8")]
55            Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
56            #[cfg(feature = "dtype-i16")]
57            Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
58            Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
59            Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
60            #[cfg(feature = "dtype-u8")]
61            UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
62            #[cfg(feature = "dtype-u16")]
63            UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
64            UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
65            UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
66            #[cfg(feature = "dtype-i128")]
67            Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
68            #[cfg(feature = "dtype-date")]
69            Date => Int32Chunked::from_chunks(name, chunks)
70                .into_date()
71                .into_series(),
72            #[cfg(feature = "dtype-time")]
73            Time => Int64Chunked::from_chunks(name, chunks)
74                .into_time()
75                .into_series(),
76            #[cfg(feature = "dtype-duration")]
77            Duration(tu) => Int64Chunked::from_chunks(name, chunks)
78                .into_duration(*tu)
79                .into_series(),
80            #[cfg(feature = "dtype-datetime")]
81            Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
82                .into_datetime(*tu, tz.clone())
83                .into_series(),
84            #[cfg(feature = "dtype-decimal")]
85            Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
86                .into_decimal_unchecked(
87                    *precision,
88                    scale.unwrap_or_else(|| unreachable!("scale should be set")),
89                )
90                .into_series(),
91            #[cfg(feature = "dtype-array")]
92            Array(_, _) => {
93                ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
94                    .into_series()
95            },
96            List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
97                .into_series(),
98            String => StringChunked::from_chunks(name, chunks).into_series(),
99            Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
100            #[cfg(feature = "dtype-categorical")]
101            dt @ (Categorical(rev_map, ordering) | Enum(rev_map, ordering)) => {
102                let cats = UInt32Chunked::from_chunks(name, chunks);
103                let rev_map = rev_map.clone().unwrap_or_else(|| {
104                    assert!(cats.is_empty());
105                    Arc::new(RevMapping::default())
106                });
107                let mut ca = CategoricalChunked::from_cats_and_rev_map_unchecked(
108                    cats,
109                    rev_map,
110                    matches!(dt, Enum(_, _)),
111                    *ordering,
112                );
113                ca.set_fast_unique(false);
114                ca.into_series()
115            },
116            Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
117            Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
118            Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
119            BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
120            #[cfg(feature = "dtype-struct")]
121            Struct(_) => {
122                let mut ca =
123                    StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
124                StructChunked::propagate_nulls_mut(&mut ca);
125                ca.into_series()
126            },
127            #[cfg(feature = "object")]
128            Object(_) => {
129                if let Some(arr) = chunks[0].as_any().downcast_ref::<FixedSizeBinaryArray>() {
130                    assert_eq!(chunks.len(), 1);
131                    // SAFETY:
132                    // this is highly unsafe. it will dereference a raw ptr on the heap
133                    // make sure the ptr is allocated and from this pid
134                    // (the pid is checked before dereference)
135                    {
136                        let pe = PolarsExtension::new(arr.clone());
137                        let s = pe.get_series(&name);
138                        pe.take_and_forget();
139                        s
140                    }
141                } else {
142                    unsafe { get_object_builder(name, 0).from_chunks(chunks) }
143                }
144            },
145            Null => new_null(name, &chunks),
146            Unknown(_) => {
147                panic!("dtype is unknown; consider supplying data-types for all operations")
148            },
149            #[allow(unreachable_patterns)]
150            _ => unreachable!(),
151        }
152    }
153
154    /// # Safety
155    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
156    pub unsafe fn _try_from_arrow_unchecked(
157        name: PlSmallStr,
158        chunks: Vec<ArrayRef>,
159        dtype: &ArrowDataType,
160    ) -> PolarsResult<Self> {
161        Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
162    }
163
164    /// Create a new Series without checking if the inner dtype of the chunks is correct
165    ///
166    /// # Safety
167    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
168    pub unsafe fn _try_from_arrow_unchecked_with_md(
169        name: PlSmallStr,
170        chunks: Vec<ArrayRef>,
171        dtype: &ArrowDataType,
172        md: Option<&Metadata>,
173    ) -> PolarsResult<Self> {
174        match dtype {
175            ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
176            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
177                let chunks =
178                    cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
179                Ok(StringChunked::from_chunks(name, chunks).into_series())
180            },
181            ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
182            ArrowDataType::LargeBinary => {
183                if let Some(md) = md {
184                    if md.maintain_type() {
185                        return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
186                    }
187                }
188                let chunks =
189                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
190                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
191            },
192            ArrowDataType::Binary => {
193                let chunks =
194                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
195                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
196            },
197            ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
198                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
199                unsafe {
200                    Ok(
201                        ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
202                            .into_series(),
203                    )
204                }
205            },
206            #[cfg(feature = "dtype-array")]
207            ArrowDataType::FixedSizeList(_, _) => {
208                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
209                unsafe {
210                    Ok(
211                        ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
212                            .into_series(),
213                    )
214                }
215            },
216            ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
217            #[cfg(feature = "dtype-u8")]
218            ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
219            #[cfg(feature = "dtype-u16")]
220            ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
221            ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
222            ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
223            #[cfg(feature = "dtype-i8")]
224            ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
225            #[cfg(feature = "dtype-i16")]
226            ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
227            ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
228            ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
229            ArrowDataType::Int128 => feature_gated!(
230                "dtype-i128",
231                Ok(Int128Chunked::from_chunks(name, chunks).into_series())
232            ),
233            ArrowDataType::Float16 => {
234                let chunks =
235                    cast_chunks(&chunks, &DataType::Float32, CastOptions::NonStrict).unwrap();
236                Ok(Float32Chunked::from_chunks(name, chunks).into_series())
237            },
238            ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
239            ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
240            #[cfg(feature = "dtype-date")]
241            ArrowDataType::Date32 => {
242                let chunks =
243                    cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
244                Ok(Int32Chunked::from_chunks(name, chunks)
245                    .into_date()
246                    .into_series())
247            },
248            #[cfg(feature = "dtype-datetime")]
249            ArrowDataType::Date64 => {
250                let chunks =
251                    cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
252                let ca = Int64Chunked::from_chunks(name, chunks);
253                Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
254            },
255            #[cfg(feature = "dtype-datetime")]
256            ArrowDataType::Timestamp(tu, tz) => {
257                let tz = TimeZone::opt_try_new(tz.clone())?;
258                let chunks =
259                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
260                let s = Int64Chunked::from_chunks(name, chunks)
261                    .into_datetime(tu.into(), tz)
262                    .into_series();
263                Ok(match tu {
264                    ArrowTimeUnit::Second => &s * MILLISECONDS,
265                    ArrowTimeUnit::Millisecond => s,
266                    ArrowTimeUnit::Microsecond => s,
267                    ArrowTimeUnit::Nanosecond => s,
268                })
269            },
270            #[cfg(feature = "dtype-duration")]
271            ArrowDataType::Duration(tu) => {
272                let chunks =
273                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
274                let s = Int64Chunked::from_chunks(name, chunks)
275                    .into_duration(tu.into())
276                    .into_series();
277                Ok(match tu {
278                    ArrowTimeUnit::Second => &s * MILLISECONDS,
279                    ArrowTimeUnit::Millisecond => s,
280                    ArrowTimeUnit::Microsecond => s,
281                    ArrowTimeUnit::Nanosecond => s,
282                })
283            },
284            #[cfg(feature = "dtype-time")]
285            ArrowDataType::Time64(tu) | ArrowDataType::Time32(tu) => {
286                let mut chunks = chunks;
287                if matches!(dtype, ArrowDataType::Time32(_)) {
288                    chunks =
289                        cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
290                }
291                let chunks =
292                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
293                let s = Int64Chunked::from_chunks(name, chunks)
294                    .into_time()
295                    .into_series();
296                Ok(match tu {
297                    ArrowTimeUnit::Second => &s * NANOSECONDS,
298                    ArrowTimeUnit::Millisecond => &s * 1_000_000,
299                    ArrowTimeUnit::Microsecond => &s * 1_000,
300                    ArrowTimeUnit::Nanosecond => s,
301                })
302            },
303            ArrowDataType::Decimal(precision, scale)
304            | ArrowDataType::Decimal256(precision, scale) => {
305                feature_gated!("dtype-decimal", {
306                    polars_ensure!(*scale <= *precision, InvalidOperation: "invalid decimal precision and scale (prec={precision}, scale={scale})");
307                    polars_ensure!(*precision <= 38, InvalidOperation: "polars does not support decimals about 38 precision");
308
309                    let mut chunks = chunks;
310                    // @NOTE: We cannot cast here as that will lower the scale.
311                    for chunk in chunks.iter_mut() {
312                        *chunk = std::mem::take(
313                            chunk
314                                .as_any_mut()
315                                .downcast_mut::<PrimitiveArray<i128>>()
316                                .unwrap(),
317                        )
318                        .to(ArrowDataType::Int128)
319                        .to_boxed();
320                    }
321                    let s = Int128Chunked::from_chunks(name, chunks)
322                        .into_decimal_unchecked(Some(*precision), *scale)
323                        .into_series();
324                    Ok(s)
325                })
326            },
327            ArrowDataType::Null => Ok(new_null(name, &chunks)),
328            #[cfg(not(feature = "dtype-categorical"))]
329            ArrowDataType::Dictionary(_, _, _) => {
330                panic!("activate dtype-categorical to convert dictionary arrays")
331            },
332            #[cfg(feature = "dtype-categorical")]
333            ArrowDataType::Dictionary(key_type, value_type, _) => {
334                use arrow::datatypes::IntegerType;
335                // don't spuriously call this; triggers a read on mmapped data
336                let arr = if chunks.len() > 1 {
337                    concatenate_unchecked(&chunks)?
338                } else {
339                    chunks[0].clone()
340                };
341
342                // If the value type is a string, they are converted to Categoricals or Enums
343                if matches!(
344                    value_type.as_ref(),
345                    ArrowDataType::Utf8
346                        | ArrowDataType::LargeUtf8
347                        | ArrowDataType::Utf8View
348                        | ArrowDataType::Null
349                ) {
350                    macro_rules! unpack_keys_values {
351                        ($dt:ty) => {{
352                            let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
353                            let keys = arr.keys();
354                            let keys = cast(keys, &ArrowDataType::UInt32).unwrap();
355                            let values = arr.values();
356                            let values = cast(&**values, &ArrowDataType::Utf8View)?;
357                            (keys, values)
358                        }};
359                    }
360
361                    use IntegerType as I;
362                    let (keys, values) = match key_type {
363                        I::Int8 => unpack_keys_values!(i8),
364                        I::UInt8 => unpack_keys_values!(u8),
365                        I::Int16 => unpack_keys_values!(i16),
366                        I::UInt16 => unpack_keys_values!(u16),
367                        I::Int32 => unpack_keys_values!(i32),
368                        I::UInt32 => unpack_keys_values!(u32),
369                        I::Int64 => unpack_keys_values!(i64),
370                        _ => polars_bail!(
371                            ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
372                        ),
373                    };
374
375                    let keys = keys.as_any().downcast_ref::<PrimitiveArray<u32>>().unwrap();
376                    let values = values.as_any().downcast_ref::<Utf8ViewArray>().unwrap();
377
378                    // Categoricals and Enums expect the RevMap values to not contain any nulls
379                    let (keys, values) =
380                        polars_compute::propagate_dictionary::propagate_dictionary_value_nulls(
381                            keys, values,
382                        );
383
384                    let mut ordering = CategoricalOrdering::default();
385                    if let Some(metadata) = md {
386                        if metadata.is_enum() {
387                            // SAFETY:
388                            // the invariants of an Arrow Dictionary guarantee the keys are in bounds
389                            return Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
390                                UInt32Chunked::with_chunk(name, keys),
391                                Arc::new(RevMapping::build_local(values)),
392                                true,
393                                CategoricalOrdering::Physical, // Enum always uses physical ordering
394                            )
395                            .into_series());
396                        } else if let Some(o) = metadata.categorical() {
397                            ordering = o;
398                        }
399                    }
400
401                    return Ok(CategoricalChunked::from_keys_and_values(
402                        name, &keys, &values, ordering,
403                    )
404                    .into_series());
405                }
406
407                macro_rules! unpack_keys_values {
408                    ($dt:ty) => {{
409                        let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
410                        let keys = arr.keys();
411                        let keys = polars_compute::cast::primitive_as_primitive::<
412                            $dt,
413                            <IdxType as PolarsNumericType>::Native,
414                        >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
415                        (arr.values(), keys)
416                    }};
417                }
418
419                use IntegerType as I;
420                let (values, keys) = match key_type {
421                    I::Int8 => unpack_keys_values!(i8),
422                    I::UInt8 => unpack_keys_values!(u8),
423                    I::Int16 => unpack_keys_values!(i16),
424                    I::UInt16 => unpack_keys_values!(u16),
425                    I::Int32 => unpack_keys_values!(i32),
426                    I::UInt32 => unpack_keys_values!(u32),
427                    I::Int64 => unpack_keys_values!(i64),
428                    _ => polars_bail!(
429                        ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
430                    ),
431                };
432
433                // Convert the dictionary to a flat array
434                let values = Series::_try_from_arrow_unchecked_with_md(
435                    name,
436                    vec![values.clone()],
437                    values.dtype(),
438                    None,
439                )?;
440                let values = values.take_unchecked(&IdxCa::from_chunks_and_dtype(
441                    PlSmallStr::EMPTY,
442                    vec![keys.to_boxed()],
443                    IDX_DTYPE,
444                ));
445
446                Ok(values)
447            },
448            #[cfg(feature = "object")]
449            ArrowDataType::Extension(ext)
450                if ext.name == EXTENSION_NAME && ext.metadata.is_some() =>
451            {
452                assert_eq!(chunks.len(), 1);
453                let arr = chunks[0]
454                    .as_any()
455                    .downcast_ref::<FixedSizeBinaryArray>()
456                    .unwrap();
457                // SAFETY:
458                // this is highly unsafe. it will dereference a raw ptr on the heap
459                // make sure the ptr is allocated and from this pid
460                // (the pid is checked before dereference)
461                let s = {
462                    let pe = PolarsExtension::new(arr.clone());
463                    let s = pe.get_series(&name);
464                    pe.take_and_forget();
465                    s
466                };
467                Ok(s)
468            },
469            #[cfg(feature = "dtype-struct")]
470            ArrowDataType::Struct(_) => {
471                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
472
473                unsafe {
474                    let mut ca =
475                        StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
476                    StructChunked::propagate_nulls_mut(&mut ca);
477                    Ok(ca.into_series())
478                }
479            },
480            ArrowDataType::FixedSizeBinary(_) => {
481                let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
482                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
483            },
484            ArrowDataType::Map(_, _) => map_arrays_to_series(name, chunks),
485            dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
486        }
487    }
488}
489
490fn map_arrays_to_series(name: PlSmallStr, chunks: Vec<ArrayRef>) -> PolarsResult<Series> {
491    let chunks = chunks
492        .iter()
493        .map(|arr| {
494            // we convert the map to the logical type: List<struct<key, value>>
495            let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
496            let inner = arr.field().clone();
497
498            // map has i32 offsets
499            let dtype = ListArray::<i32>::default_datatype(inner.dtype().clone());
500            Box::new(ListArray::<i32>::new(
501                dtype,
502                arr.offsets().clone(),
503                inner,
504                arr.validity().cloned(),
505            )) as ArrayRef
506        })
507        .collect::<Vec<_>>();
508    Series::try_from((name, chunks))
509}
510
511fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
512    arr.iter().map(|arr| f(&**arr)).collect()
513}
514
515/// Converts to physical types and bubbles up the correct [`DataType`].
516#[allow(clippy::only_used_in_recursion)]
517unsafe fn to_physical_and_dtype(
518    arrays: Vec<ArrayRef>,
519    md: Option<&Metadata>,
520) -> (Vec<ArrayRef>, DataType) {
521    match arrays[0].dtype() {
522        ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
523            let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
524            (chunks, DataType::String)
525        },
526        ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
527            let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
528            (chunks, DataType::Binary)
529        },
530        #[allow(unused_variables)]
531        dt @ ArrowDataType::Dictionary(_, _, _) => {
532            feature_gated!("dtype-categorical", {
533                let s = unsafe {
534                    let dt = dt.clone();
535                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
536                }
537                .unwrap();
538                (s.chunks().clone(), s.dtype().clone())
539            })
540        },
541        ArrowDataType::List(field) => {
542            let out = convert(&arrays, |arr| {
543                cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
544            });
545            to_physical_and_dtype(out, md)
546        },
547        #[cfg(feature = "dtype-array")]
548        ArrowDataType::FixedSizeList(field, size) => {
549            let values = arrays
550                .iter()
551                .map(|arr| {
552                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
553                    arr.values().clone()
554                })
555                .collect::<Vec<_>>();
556
557            let (converted_values, dtype) =
558                to_physical_and_dtype(values, field.metadata.as_deref());
559
560            let arrays = arrays
561                .iter()
562                .zip(converted_values)
563                .map(|(arr, values)| {
564                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
565
566                    let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
567                    Box::from(FixedSizeListArray::new(
568                        dtype,
569                        arr.len(),
570                        values,
571                        arr.validity().cloned(),
572                    )) as ArrayRef
573                })
574                .collect();
575            (arrays, DataType::Array(Box::new(dtype), *size))
576        },
577        ArrowDataType::LargeList(field) => {
578            let values = arrays
579                .iter()
580                .map(|arr| {
581                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
582                    arr.values().clone()
583                })
584                .collect::<Vec<_>>();
585
586            let (converted_values, dtype) =
587                to_physical_and_dtype(values, field.metadata.as_deref());
588
589            let arrays = arrays
590                .iter()
591                .zip(converted_values)
592                .map(|(arr, values)| {
593                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
594
595                    let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
596                    Box::from(ListArray::<i64>::new(
597                        dtype,
598                        arr.offsets().clone(),
599                        values,
600                        arr.validity().cloned(),
601                    )) as ArrayRef
602                })
603                .collect();
604            (arrays, DataType::List(Box::new(dtype)))
605        },
606        ArrowDataType::Struct(_fields) => {
607            feature_gated!("dtype-struct", {
608                let mut pl_fields = None;
609                let arrays = arrays
610                    .iter()
611                    .map(|arr| {
612                        let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
613                        let (values, dtypes): (Vec<_>, Vec<_>) = arr
614                            .values()
615                            .iter()
616                            .zip(_fields.iter())
617                            .map(|(value, field)| {
618                                let mut out = to_physical_and_dtype(
619                                    vec![value.clone()],
620                                    field.metadata.as_deref(),
621                                );
622                                (out.0.pop().unwrap(), out.1)
623                            })
624                            .unzip();
625
626                        let arrow_fields = values
627                            .iter()
628                            .zip(_fields.iter())
629                            .map(|(arr, field)| {
630                                ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
631                            })
632                            .collect();
633                        let arrow_array = Box::new(StructArray::new(
634                            ArrowDataType::Struct(arrow_fields),
635                            arr.len(),
636                            values,
637                            arr.validity().cloned(),
638                        )) as ArrayRef;
639
640                        if pl_fields.is_none() {
641                            pl_fields = Some(
642                                _fields
643                                    .iter()
644                                    .zip(dtypes)
645                                    .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
646                                    .collect_vec(),
647                            )
648                        }
649
650                        arrow_array
651                    })
652                    .collect_vec();
653
654                (arrays, DataType::Struct(pl_fields.unwrap()))
655            })
656        },
657        // Use Series architecture to convert nested logical types to physical.
658        dt @ (ArrowDataType::Duration(_)
659        | ArrowDataType::Time32(_)
660        | ArrowDataType::Time64(_)
661        | ArrowDataType::Timestamp(_, _)
662        | ArrowDataType::Date32
663        | ArrowDataType::Decimal(_, _)
664        | ArrowDataType::Date64) => {
665            let dt = dt.clone();
666            let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt).unwrap();
667            let dtype = s.dtype().clone();
668            (std::mem::take(s.chunks_mut()), dtype)
669        },
670        dt => {
671            let dtype = DataType::from_arrow(dt, md);
672            (arrays, dtype)
673        },
674    }
675}
676
677fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
678    let mut chunks_iter = chunks.iter();
679    let dtype: ArrowDataType = chunks_iter
680        .next()
681        .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
682        .dtype()
683        .clone();
684
685    for chunk in chunks_iter {
686        if chunk.dtype() != &dtype {
687            polars_bail!(
688                ComputeError: "cannot create series from multiple arrays with different types"
689            );
690        }
691    }
692    Ok(dtype)
693}
694
695impl Series {
696    pub fn try_new<T>(
697        name: PlSmallStr,
698        data: T,
699    ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
700    where
701        (PlSmallStr, T): TryInto<Self>,
702    {
703        // # TODO
704        // * Remove the TryFrom<tuple> impls in favor of this
705        <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
706    }
707}
708
709impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
710    type Error = PolarsError;
711
712    fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
713        let (name, chunks) = name_arr;
714
715        let dtype = check_types(&chunks)?;
716        // SAFETY:
717        // dtype is checked
718        unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
719    }
720}
721
722impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
723    type Error = PolarsError;
724
725    fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
726        let (name, arr) = name_arr;
727        Series::try_from((name, vec![arr]))
728    }
729}
730
731impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
732    type Error = PolarsError;
733
734    fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
735        let (field, chunks) = field_arr;
736
737        let dtype = check_types(&chunks)?;
738
739        // SAFETY:
740        // dtype is checked
741        unsafe {
742            Series::_try_from_arrow_unchecked_with_md(
743                field.name.clone(),
744                chunks,
745                &dtype,
746                field.metadata.as_deref(),
747            )
748        }
749    }
750}
751
752impl TryFrom<(&ArrowField, ArrayRef)> for Series {
753    type Error = PolarsError;
754
755    fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
756        let (field, arr) = field_arr;
757        Series::try_from((field, vec![arr]))
758    }
759}
760
761/// Used to convert a [`ChunkedArray`], `&dyn SeriesTrait` and [`Series`]
762/// into a [`Series`].
763/// # Safety
764///
765/// This trait is marked `unsafe` as the `is_series` return is used
766/// to transmute to `Series`. This must always return `false` except
767/// for `Series` structs.
768pub unsafe trait IntoSeries {
769    fn is_series() -> bool {
770        false
771    }
772
773    fn into_series(self) -> Series
774    where
775        Self: Sized;
776}
777
778impl<T> From<ChunkedArray<T>> for Series
779where
780    T: PolarsDataType,
781    ChunkedArray<T>: IntoSeries,
782{
783    fn from(ca: ChunkedArray<T>) -> Self {
784        ca.into_series()
785    }
786}
787
788#[cfg(feature = "dtype-date")]
789impl From<DateChunked> for Series {
790    fn from(a: DateChunked) -> Self {
791        a.into_series()
792    }
793}
794
795#[cfg(feature = "dtype-datetime")]
796impl From<DatetimeChunked> for Series {
797    fn from(a: DatetimeChunked) -> Self {
798        a.into_series()
799    }
800}
801
802#[cfg(feature = "dtype-duration")]
803impl From<DurationChunked> for Series {
804    fn from(a: DurationChunked) -> Self {
805        a.into_series()
806    }
807}
808
809#[cfg(feature = "dtype-time")]
810impl From<TimeChunked> for Series {
811    fn from(a: TimeChunked) -> Self {
812        a.into_series()
813    }
814}
815
816unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
817    fn into_series(self) -> Series {
818        Series(self)
819    }
820}
821
822unsafe impl IntoSeries for Series {
823    fn is_series() -> bool {
824        true
825    }
826
827    fn into_series(self) -> Series {
828        self
829    }
830}
831
832fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
833    let len = chunks.iter().map(|arr| arr.len()).sum();
834    Series::new_null(name, len)
835}
polars_core/series/from.rs

polars_core/series/
from.rs
