polars_core/series/
from.rs

1#[cfg(feature = "dtype-categorical")]
2use arrow::compute::concatenate::concatenate_unchecked;
3use arrow::datatypes::Metadata;
4#[cfg(any(
5    feature = "dtype-date",
6    feature = "dtype-datetime",
7    feature = "dtype-time",
8    feature = "dtype-duration"
9))]
10use arrow::temporal_conversions::*;
11use polars_compute::cast::cast_unchecked as cast;
12use polars_error::feature_gated;
13use polars_utils::itertools::Itertools;
14
15use crate::chunked_array::cast::{CastOptions, cast_chunks};
16#[cfg(feature = "object")]
17use crate::chunked_array::object::extension::polars_extension::PolarsExtension;
18#[cfg(feature = "object")]
19use crate::chunked_array::object::registry::get_object_builder;
20#[cfg(feature = "timezones")]
21use crate::chunked_array::temporal::parse_fixed_offset;
22#[cfg(feature = "timezones")]
23use crate::chunked_array::temporal::validate_time_zone;
24use crate::prelude::*;
25
26impl Series {
27    pub fn from_chunk_and_dtype(
28        name: PlSmallStr,
29        chunk: ArrayRef,
30        dtype: &DataType,
31    ) -> PolarsResult<Self> {
32        if &dtype.to_physical().to_arrow(CompatLevel::newest()) != chunk.dtype() {
33            polars_bail!(
34                InvalidOperation: "cannot create a series of type '{dtype}' of arrow chunk with type '{:?}'",
35                chunk.dtype()
36            );
37        }
38
39        // SAFETY: We check that the datatype matches.
40        let series = unsafe { Self::from_chunks_and_dtype_unchecked(name, vec![chunk], dtype) };
41        Ok(series)
42    }
43
44    /// Takes chunks and a polars datatype and constructs the Series
45    /// This is faster than creating from chunks and an arrow datatype because there is no
46    /// casting involved
47    ///
48    /// # Safety
49    ///
50    /// The caller must ensure that the given `dtype`'s physical type matches all the `ArrayRef` dtypes.
51    pub unsafe fn from_chunks_and_dtype_unchecked(
52        name: PlSmallStr,
53        chunks: Vec<ArrayRef>,
54        dtype: &DataType,
55    ) -> Self {
56        use DataType::*;
57        match dtype {
58            #[cfg(feature = "dtype-i8")]
59            Int8 => Int8Chunked::from_chunks(name, chunks).into_series(),
60            #[cfg(feature = "dtype-i16")]
61            Int16 => Int16Chunked::from_chunks(name, chunks).into_series(),
62            Int32 => Int32Chunked::from_chunks(name, chunks).into_series(),
63            Int64 => Int64Chunked::from_chunks(name, chunks).into_series(),
64            #[cfg(feature = "dtype-u8")]
65            UInt8 => UInt8Chunked::from_chunks(name, chunks).into_series(),
66            #[cfg(feature = "dtype-u16")]
67            UInt16 => UInt16Chunked::from_chunks(name, chunks).into_series(),
68            UInt32 => UInt32Chunked::from_chunks(name, chunks).into_series(),
69            UInt64 => UInt64Chunked::from_chunks(name, chunks).into_series(),
70            #[cfg(feature = "dtype-i128")]
71            Int128 => Int128Chunked::from_chunks(name, chunks).into_series(),
72            #[cfg(feature = "dtype-date")]
73            Date => Int32Chunked::from_chunks(name, chunks)
74                .into_date()
75                .into_series(),
76            #[cfg(feature = "dtype-time")]
77            Time => Int64Chunked::from_chunks(name, chunks)
78                .into_time()
79                .into_series(),
80            #[cfg(feature = "dtype-duration")]
81            Duration(tu) => Int64Chunked::from_chunks(name, chunks)
82                .into_duration(*tu)
83                .into_series(),
84            #[cfg(feature = "dtype-datetime")]
85            Datetime(tu, tz) => Int64Chunked::from_chunks(name, chunks)
86                .into_datetime(*tu, tz.clone())
87                .into_series(),
88            #[cfg(feature = "dtype-decimal")]
89            Decimal(precision, scale) => Int128Chunked::from_chunks(name, chunks)
90                .into_decimal_unchecked(
91                    *precision,
92                    scale.unwrap_or_else(|| unreachable!("scale should be set")),
93                )
94                .into_series(),
95            #[cfg(feature = "dtype-array")]
96            Array(_, _) => {
97                ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
98                    .into_series()
99            },
100            List(_) => ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone())
101                .into_series(),
102            String => StringChunked::from_chunks(name, chunks).into_series(),
103            Binary => BinaryChunked::from_chunks(name, chunks).into_series(),
104            #[cfg(feature = "dtype-categorical")]
105            dt @ (Categorical(rev_map, ordering) | Enum(rev_map, ordering)) => {
106                let cats = UInt32Chunked::from_chunks(name, chunks);
107                let rev_map = rev_map.clone().unwrap_or_else(|| {
108                    assert!(cats.is_empty());
109                    Arc::new(RevMapping::default())
110                });
111                let mut ca = CategoricalChunked::from_cats_and_rev_map_unchecked(
112                    cats,
113                    rev_map,
114                    matches!(dt, Enum(_, _)),
115                    *ordering,
116                );
117                ca.set_fast_unique(false);
118                ca.into_series()
119            },
120            Boolean => BooleanChunked::from_chunks(name, chunks).into_series(),
121            Float32 => Float32Chunked::from_chunks(name, chunks).into_series(),
122            Float64 => Float64Chunked::from_chunks(name, chunks).into_series(),
123            BinaryOffset => BinaryOffsetChunked::from_chunks(name, chunks).into_series(),
124            #[cfg(feature = "dtype-struct")]
125            Struct(_) => {
126                let mut ca =
127                    StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype.clone());
128                StructChunked::propagate_nulls_mut(&mut ca);
129                ca.into_series()
130            },
131            #[cfg(feature = "object")]
132            Object(_) => {
133                if let Some(arr) = chunks[0].as_any().downcast_ref::<FixedSizeBinaryArray>() {
134                    assert_eq!(chunks.len(), 1);
135                    // SAFETY:
136                    // this is highly unsafe. it will dereference a raw ptr on the heap
137                    // make sure the ptr is allocated and from this pid
138                    // (the pid is checked before dereference)
139                    {
140                        let pe = PolarsExtension::new(arr.clone());
141                        let s = pe.get_series(&name);
142                        pe.take_and_forget();
143                        s
144                    }
145                } else {
146                    unsafe { get_object_builder(name, 0).from_chunks(chunks) }
147                }
148            },
149            Null => new_null(name, &chunks),
150            Unknown(_) => {
151                panic!("dtype is unknown; consider supplying data-types for all operations")
152            },
153            #[allow(unreachable_patterns)]
154            _ => unreachable!(),
155        }
156    }
157
158    /// # Safety
159    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
160    pub unsafe fn _try_from_arrow_unchecked(
161        name: PlSmallStr,
162        chunks: Vec<ArrayRef>,
163        dtype: &ArrowDataType,
164    ) -> PolarsResult<Self> {
165        Self::_try_from_arrow_unchecked_with_md(name, chunks, dtype, None)
166    }
167
168    /// Create a new Series without checking if the inner dtype of the chunks is correct
169    ///
170    /// # Safety
171    /// The caller must ensure that the given `dtype` matches all the `ArrayRef` dtypes.
172    pub unsafe fn _try_from_arrow_unchecked_with_md(
173        name: PlSmallStr,
174        chunks: Vec<ArrayRef>,
175        dtype: &ArrowDataType,
176        md: Option<&Metadata>,
177    ) -> PolarsResult<Self> {
178        match dtype {
179            ArrowDataType::Utf8View => Ok(StringChunked::from_chunks(name, chunks).into_series()),
180            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
181                let chunks =
182                    cast_chunks(&chunks, &DataType::String, CastOptions::NonStrict).unwrap();
183                Ok(StringChunked::from_chunks(name, chunks).into_series())
184            },
185            ArrowDataType::BinaryView => Ok(BinaryChunked::from_chunks(name, chunks).into_series()),
186            ArrowDataType::LargeBinary => {
187                if let Some(md) = md {
188                    if md.maintain_type() {
189                        return Ok(BinaryOffsetChunked::from_chunks(name, chunks).into_series());
190                    }
191                }
192                let chunks =
193                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
194                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
195            },
196            ArrowDataType::Binary => {
197                let chunks =
198                    cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict).unwrap();
199                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
200            },
201            ArrowDataType::List(_) | ArrowDataType::LargeList(_) => {
202                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
203                unsafe {
204                    Ok(
205                        ListChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
206                            .into_series(),
207                    )
208                }
209            },
210            #[cfg(feature = "dtype-array")]
211            ArrowDataType::FixedSizeList(_, _) => {
212                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
213                unsafe {
214                    Ok(
215                        ArrayChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype)
216                            .into_series(),
217                    )
218                }
219            },
220            ArrowDataType::Boolean => Ok(BooleanChunked::from_chunks(name, chunks).into_series()),
221            #[cfg(feature = "dtype-u8")]
222            ArrowDataType::UInt8 => Ok(UInt8Chunked::from_chunks(name, chunks).into_series()),
223            #[cfg(feature = "dtype-u16")]
224            ArrowDataType::UInt16 => Ok(UInt16Chunked::from_chunks(name, chunks).into_series()),
225            ArrowDataType::UInt32 => Ok(UInt32Chunked::from_chunks(name, chunks).into_series()),
226            ArrowDataType::UInt64 => Ok(UInt64Chunked::from_chunks(name, chunks).into_series()),
227            #[cfg(feature = "dtype-i8")]
228            ArrowDataType::Int8 => Ok(Int8Chunked::from_chunks(name, chunks).into_series()),
229            #[cfg(feature = "dtype-i16")]
230            ArrowDataType::Int16 => Ok(Int16Chunked::from_chunks(name, chunks).into_series()),
231            ArrowDataType::Int32 => Ok(Int32Chunked::from_chunks(name, chunks).into_series()),
232            ArrowDataType::Int64 => Ok(Int64Chunked::from_chunks(name, chunks).into_series()),
233            ArrowDataType::Int128 => feature_gated!(
234                "dtype-i128",
235                Ok(Int128Chunked::from_chunks(name, chunks).into_series())
236            ),
237            ArrowDataType::Float16 => {
238                let chunks =
239                    cast_chunks(&chunks, &DataType::Float32, CastOptions::NonStrict).unwrap();
240                Ok(Float32Chunked::from_chunks(name, chunks).into_series())
241            },
242            ArrowDataType::Float32 => Ok(Float32Chunked::from_chunks(name, chunks).into_series()),
243            ArrowDataType::Float64 => Ok(Float64Chunked::from_chunks(name, chunks).into_series()),
244            #[cfg(feature = "dtype-date")]
245            ArrowDataType::Date32 => {
246                let chunks =
247                    cast_chunks(&chunks, &DataType::Int32, CastOptions::Overflowing).unwrap();
248                Ok(Int32Chunked::from_chunks(name, chunks)
249                    .into_date()
250                    .into_series())
251            },
252            #[cfg(feature = "dtype-datetime")]
253            ArrowDataType::Date64 => {
254                let chunks =
255                    cast_chunks(&chunks, &DataType::Int64, CastOptions::Overflowing).unwrap();
256                let ca = Int64Chunked::from_chunks(name, chunks);
257                Ok(ca.into_datetime(TimeUnit::Milliseconds, None).into_series())
258            },
259            #[cfg(feature = "dtype-datetime")]
260            ArrowDataType::Timestamp(tu, tz) => {
261                let canonical_tz = DataType::canonical_timezone(tz);
262                let tz = match canonical_tz.as_deref() {
263                    #[cfg(feature = "timezones")]
264                    Some(tz_str) => match validate_time_zone(tz_str) {
265                        Ok(_) => canonical_tz,
266                        Err(_) => Some(parse_fixed_offset(tz_str)?),
267                    },
268                    _ => canonical_tz,
269                };
270                let chunks =
271                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
272                let s = Int64Chunked::from_chunks(name, chunks)
273                    .into_datetime(tu.into(), tz)
274                    .into_series();
275                Ok(match tu {
276                    ArrowTimeUnit::Second => &s * MILLISECONDS,
277                    ArrowTimeUnit::Millisecond => s,
278                    ArrowTimeUnit::Microsecond => s,
279                    ArrowTimeUnit::Nanosecond => s,
280                })
281            },
282            #[cfg(feature = "dtype-duration")]
283            ArrowDataType::Duration(tu) => {
284                let chunks =
285                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
286                let s = Int64Chunked::from_chunks(name, chunks)
287                    .into_duration(tu.into())
288                    .into_series();
289                Ok(match tu {
290                    ArrowTimeUnit::Second => &s * MILLISECONDS,
291                    ArrowTimeUnit::Millisecond => s,
292                    ArrowTimeUnit::Microsecond => s,
293                    ArrowTimeUnit::Nanosecond => s,
294                })
295            },
296            #[cfg(feature = "dtype-time")]
297            ArrowDataType::Time64(tu) | ArrowDataType::Time32(tu) => {
298                let mut chunks = chunks;
299                if matches!(dtype, ArrowDataType::Time32(_)) {
300                    chunks =
301                        cast_chunks(&chunks, &DataType::Int32, CastOptions::NonStrict).unwrap();
302                }
303                let chunks =
304                    cast_chunks(&chunks, &DataType::Int64, CastOptions::NonStrict).unwrap();
305                let s = Int64Chunked::from_chunks(name, chunks)
306                    .into_time()
307                    .into_series();
308                Ok(match tu {
309                    ArrowTimeUnit::Second => &s * NANOSECONDS,
310                    ArrowTimeUnit::Millisecond => &s * 1_000_000,
311                    ArrowTimeUnit::Microsecond => &s * 1_000,
312                    ArrowTimeUnit::Nanosecond => s,
313                })
314            },
315            ArrowDataType::Decimal(precision, scale)
316            | ArrowDataType::Decimal256(precision, scale) => {
317                feature_gated!("dtype-decimal", {
318                    polars_ensure!(*scale <= *precision, InvalidOperation: "invalid decimal precision and scale (prec={precision}, scale={scale})");
319                    polars_ensure!(*precision <= 38, InvalidOperation: "polars does not support decimals about 38 precision");
320
321                    let mut chunks = chunks;
322                    // @NOTE: We cannot cast here as that will lower the scale.
323                    for chunk in chunks.iter_mut() {
324                        *chunk = std::mem::take(
325                            chunk
326                                .as_any_mut()
327                                .downcast_mut::<PrimitiveArray<i128>>()
328                                .unwrap(),
329                        )
330                        .to(ArrowDataType::Int128)
331                        .to_boxed();
332                    }
333                    let s = Int128Chunked::from_chunks(name, chunks)
334                        .into_decimal_unchecked(Some(*precision), *scale)
335                        .into_series();
336                    Ok(s)
337                })
338            },
339            ArrowDataType::Null => Ok(new_null(name, &chunks)),
340            #[cfg(not(feature = "dtype-categorical"))]
341            ArrowDataType::Dictionary(_, _, _) => {
342                panic!("activate dtype-categorical to convert dictionary arrays")
343            },
344            #[cfg(feature = "dtype-categorical")]
345            ArrowDataType::Dictionary(key_type, value_type, _) => {
346                use arrow::datatypes::IntegerType;
347                // don't spuriously call this; triggers a read on mmapped data
348                let arr = if chunks.len() > 1 {
349                    concatenate_unchecked(&chunks)?
350                } else {
351                    chunks[0].clone()
352                };
353
354                // If the value type is a string, they are converted to Categoricals or Enums
355                if matches!(
356                    value_type.as_ref(),
357                    ArrowDataType::Utf8
358                        | ArrowDataType::LargeUtf8
359                        | ArrowDataType::Utf8View
360                        | ArrowDataType::Null
361                ) {
362                    macro_rules! unpack_keys_values {
363                        ($dt:ty) => {{
364                            let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
365                            let keys = arr.keys();
366                            let keys = cast(keys, &ArrowDataType::UInt32).unwrap();
367                            let values = arr.values();
368                            let values = cast(&**values, &ArrowDataType::Utf8View)?;
369                            (keys, values)
370                        }};
371                    }
372
373                    use IntegerType as I;
374                    let (keys, values) = match key_type {
375                        I::Int8 => unpack_keys_values!(i8),
376                        I::UInt8 => unpack_keys_values!(u8),
377                        I::Int16 => unpack_keys_values!(i16),
378                        I::UInt16 => unpack_keys_values!(u16),
379                        I::Int32 => unpack_keys_values!(i32),
380                        I::UInt32 => unpack_keys_values!(u32),
381                        I::Int64 => unpack_keys_values!(i64),
382                        _ => polars_bail!(
383                            ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
384                        ),
385                    };
386
387                    let keys = keys.as_any().downcast_ref::<PrimitiveArray<u32>>().unwrap();
388                    let values = values.as_any().downcast_ref::<Utf8ViewArray>().unwrap();
389
390                    // Categoricals and Enums expect the RevMap values to not contain any nulls
391                    let (keys, values) =
392                        polars_compute::propagate_dictionary::propagate_dictionary_value_nulls(
393                            keys, values,
394                        );
395
396                    let mut ordering = CategoricalOrdering::default();
397                    if let Some(metadata) = md {
398                        if metadata.is_enum() {
399                            // SAFETY:
400                            // the invariants of an Arrow Dictionary guarantee the keys are in bounds
401                            return Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
402                                UInt32Chunked::with_chunk(name, keys),
403                                Arc::new(RevMapping::build_local(values)),
404                                true,
405                                CategoricalOrdering::Physical, // Enum always uses physical ordering
406                            )
407                            .into_series());
408                        } else if let Some(o) = metadata.categorical() {
409                            ordering = o;
410                        }
411                    }
412
413                    return Ok(CategoricalChunked::from_keys_and_values(
414                        name, &keys, &values, ordering,
415                    )
416                    .into_series());
417                }
418
419                macro_rules! unpack_keys_values {
420                    ($dt:ty) => {{
421                        let arr = arr.as_any().downcast_ref::<DictionaryArray<$dt>>().unwrap();
422                        let keys = arr.keys();
423                        let keys = polars_compute::cast::primitive_as_primitive::<
424                            $dt,
425                            <IdxType as PolarsNumericType>::Native,
426                        >(keys, &IDX_DTYPE.to_arrow(CompatLevel::newest()));
427                        (arr.values(), keys)
428                    }};
429                }
430
431                use IntegerType as I;
432                let (values, keys) = match key_type {
433                    I::Int8 => unpack_keys_values!(i8),
434                    I::UInt8 => unpack_keys_values!(u8),
435                    I::Int16 => unpack_keys_values!(i16),
436                    I::UInt16 => unpack_keys_values!(u16),
437                    I::Int32 => unpack_keys_values!(i32),
438                    I::UInt32 => unpack_keys_values!(u32),
439                    I::Int64 => unpack_keys_values!(i64),
440                    _ => polars_bail!(
441                        ComputeError: "dictionaries with unsigned 64-bit keys are not supported"
442                    ),
443                };
444
445                // Convert the dictionary to a flat array
446                let values = Series::_try_from_arrow_unchecked_with_md(
447                    name,
448                    vec![values.clone()],
449                    values.dtype(),
450                    None,
451                )?;
452                let values = values.take_unchecked(&IdxCa::from_chunks_and_dtype(
453                    PlSmallStr::EMPTY,
454                    vec![keys.to_boxed()],
455                    IDX_DTYPE,
456                ));
457
458                Ok(values)
459            },
460            #[cfg(feature = "object")]
461            ArrowDataType::Extension(ext)
462                if ext.name == EXTENSION_NAME && ext.metadata.is_some() =>
463            {
464                assert_eq!(chunks.len(), 1);
465                let arr = chunks[0]
466                    .as_any()
467                    .downcast_ref::<FixedSizeBinaryArray>()
468                    .unwrap();
469                // SAFETY:
470                // this is highly unsafe. it will dereference a raw ptr on the heap
471                // make sure the ptr is allocated and from this pid
472                // (the pid is checked before dereference)
473                let s = {
474                    let pe = PolarsExtension::new(arr.clone());
475                    let s = pe.get_series(&name);
476                    pe.take_and_forget();
477                    s
478                };
479                Ok(s)
480            },
481            #[cfg(feature = "dtype-struct")]
482            ArrowDataType::Struct(_) => {
483                let (chunks, dtype) = to_physical_and_dtype(chunks, md);
484
485                unsafe {
486                    let mut ca =
487                        StructChunked::from_chunks_and_dtype_unchecked(name, chunks, dtype);
488                    StructChunked::propagate_nulls_mut(&mut ca);
489                    Ok(ca.into_series())
490                }
491            },
492            ArrowDataType::FixedSizeBinary(_) => {
493                let chunks = cast_chunks(&chunks, &DataType::Binary, CastOptions::NonStrict)?;
494                Ok(BinaryChunked::from_chunks(name, chunks).into_series())
495            },
496            ArrowDataType::Map(_, _) => map_arrays_to_series(name, chunks),
497            dt => polars_bail!(ComputeError: "cannot create series from {:?}", dt),
498        }
499    }
500}
501
502fn map_arrays_to_series(name: PlSmallStr, chunks: Vec<ArrayRef>) -> PolarsResult<Series> {
503    let chunks = chunks
504        .iter()
505        .map(|arr| {
506            // we convert the map to the logical type: List<struct<key, value>>
507            let arr = arr.as_any().downcast_ref::<MapArray>().unwrap();
508            let inner = arr.field().clone();
509
510            // map has i32 offsets
511            let dtype = ListArray::<i32>::default_datatype(inner.dtype().clone());
512            Box::new(ListArray::<i32>::new(
513                dtype,
514                arr.offsets().clone(),
515                inner,
516                arr.validity().cloned(),
517            )) as ArrayRef
518        })
519        .collect::<Vec<_>>();
520    Series::try_from((name, chunks))
521}
522
523fn convert<F: Fn(&dyn Array) -> ArrayRef>(arr: &[ArrayRef], f: F) -> Vec<ArrayRef> {
524    arr.iter().map(|arr| f(&**arr)).collect()
525}
526
527/// Converts to physical types and bubbles up the correct [`DataType`].
528#[allow(clippy::only_used_in_recursion)]
529unsafe fn to_physical_and_dtype(
530    arrays: Vec<ArrayRef>,
531    md: Option<&Metadata>,
532) -> (Vec<ArrayRef>, DataType) {
533    match arrays[0].dtype() {
534        ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 => {
535            let chunks = cast_chunks(&arrays, &DataType::String, CastOptions::NonStrict).unwrap();
536            (chunks, DataType::String)
537        },
538        ArrowDataType::Binary | ArrowDataType::LargeBinary | ArrowDataType::FixedSizeBinary(_) => {
539            let chunks = cast_chunks(&arrays, &DataType::Binary, CastOptions::NonStrict).unwrap();
540            (chunks, DataType::Binary)
541        },
542        #[allow(unused_variables)]
543        dt @ ArrowDataType::Dictionary(_, _, _) => {
544            feature_gated!("dtype-categorical", {
545                let s = unsafe {
546                    let dt = dt.clone();
547                    Series::_try_from_arrow_unchecked_with_md(PlSmallStr::EMPTY, arrays, &dt, md)
548                }
549                .unwrap();
550                (s.chunks().clone(), s.dtype().clone())
551            })
552        },
553        ArrowDataType::List(field) => {
554            let out = convert(&arrays, |arr| {
555                cast(arr, &ArrowDataType::LargeList(field.clone())).unwrap()
556            });
557            to_physical_and_dtype(out, md)
558        },
559        #[cfg(feature = "dtype-array")]
560        ArrowDataType::FixedSizeList(field, size) => {
561            let values = arrays
562                .iter()
563                .map(|arr| {
564                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
565                    arr.values().clone()
566                })
567                .collect::<Vec<_>>();
568
569            let (converted_values, dtype) =
570                to_physical_and_dtype(values, field.metadata.as_deref());
571
572            let arrays = arrays
573                .iter()
574                .zip(converted_values)
575                .map(|(arr, values)| {
576                    let arr = arr.as_any().downcast_ref::<FixedSizeListArray>().unwrap();
577
578                    let dtype = FixedSizeListArray::default_datatype(values.dtype().clone(), *size);
579                    Box::from(FixedSizeListArray::new(
580                        dtype,
581                        arr.len(),
582                        values,
583                        arr.validity().cloned(),
584                    )) as ArrayRef
585                })
586                .collect();
587            (arrays, DataType::Array(Box::new(dtype), *size))
588        },
589        ArrowDataType::LargeList(field) => {
590            let values = arrays
591                .iter()
592                .map(|arr| {
593                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
594                    arr.values().clone()
595                })
596                .collect::<Vec<_>>();
597
598            let (converted_values, dtype) =
599                to_physical_and_dtype(values, field.metadata.as_deref());
600
601            let arrays = arrays
602                .iter()
603                .zip(converted_values)
604                .map(|(arr, values)| {
605                    let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();
606
607                    let dtype = ListArray::<i64>::default_datatype(values.dtype().clone());
608                    Box::from(ListArray::<i64>::new(
609                        dtype,
610                        arr.offsets().clone(),
611                        values,
612                        arr.validity().cloned(),
613                    )) as ArrayRef
614                })
615                .collect();
616            (arrays, DataType::List(Box::new(dtype)))
617        },
618        ArrowDataType::Struct(_fields) => {
619            feature_gated!("dtype-struct", {
620                let mut pl_fields = None;
621                let arrays = arrays
622                    .iter()
623                    .map(|arr| {
624                        let arr = arr.as_any().downcast_ref::<StructArray>().unwrap();
625                        let (values, dtypes): (Vec<_>, Vec<_>) = arr
626                            .values()
627                            .iter()
628                            .zip(_fields.iter())
629                            .map(|(value, field)| {
630                                let mut out = to_physical_and_dtype(
631                                    vec![value.clone()],
632                                    field.metadata.as_deref(),
633                                );
634                                (out.0.pop().unwrap(), out.1)
635                            })
636                            .unzip();
637
638                        let arrow_fields = values
639                            .iter()
640                            .zip(_fields.iter())
641                            .map(|(arr, field)| {
642                                ArrowField::new(field.name.clone(), arr.dtype().clone(), true)
643                            })
644                            .collect();
645                        let arrow_array = Box::new(StructArray::new(
646                            ArrowDataType::Struct(arrow_fields),
647                            arr.len(),
648                            values,
649                            arr.validity().cloned(),
650                        )) as ArrayRef;
651
652                        if pl_fields.is_none() {
653                            pl_fields = Some(
654                                _fields
655                                    .iter()
656                                    .zip(dtypes)
657                                    .map(|(field, dtype)| Field::new(field.name.clone(), dtype))
658                                    .collect_vec(),
659                            )
660                        }
661
662                        arrow_array
663                    })
664                    .collect_vec();
665
666                (arrays, DataType::Struct(pl_fields.unwrap()))
667            })
668        },
669        // Use Series architecture to convert nested logical types to physical.
670        dt @ (ArrowDataType::Duration(_)
671        | ArrowDataType::Time32(_)
672        | ArrowDataType::Time64(_)
673        | ArrowDataType::Timestamp(_, _)
674        | ArrowDataType::Date32
675        | ArrowDataType::Decimal(_, _)
676        | ArrowDataType::Date64) => {
677            let dt = dt.clone();
678            let mut s = Series::_try_from_arrow_unchecked(PlSmallStr::EMPTY, arrays, &dt).unwrap();
679            let dtype = s.dtype().clone();
680            (std::mem::take(s.chunks_mut()), dtype)
681        },
682        dt => {
683            let dtype = DataType::from_arrow(dt, true, md);
684            (arrays, dtype)
685        },
686    }
687}
688
689fn check_types(chunks: &[ArrayRef]) -> PolarsResult<ArrowDataType> {
690    let mut chunks_iter = chunks.iter();
691    let dtype: ArrowDataType = chunks_iter
692        .next()
693        .ok_or_else(|| polars_err!(NoData: "expected at least one array-ref"))?
694        .dtype()
695        .clone();
696
697    for chunk in chunks_iter {
698        if chunk.dtype() != &dtype {
699            polars_bail!(
700                ComputeError: "cannot create series from multiple arrays with different types"
701            );
702        }
703    }
704    Ok(dtype)
705}
706
707impl Series {
708    pub fn try_new<T>(
709        name: PlSmallStr,
710        data: T,
711    ) -> Result<Self, <(PlSmallStr, T) as TryInto<Self>>::Error>
712    where
713        (PlSmallStr, T): TryInto<Self>,
714    {
715        // # TODO
716        // * Remove the TryFrom<tuple> impls in favor of this
717        <(PlSmallStr, T) as TryInto<Self>>::try_into((name, data))
718    }
719}
720
721impl TryFrom<(PlSmallStr, Vec<ArrayRef>)> for Series {
722    type Error = PolarsError;
723
724    fn try_from(name_arr: (PlSmallStr, Vec<ArrayRef>)) -> PolarsResult<Self> {
725        let (name, chunks) = name_arr;
726
727        let dtype = check_types(&chunks)?;
728        // SAFETY:
729        // dtype is checked
730        unsafe { Series::_try_from_arrow_unchecked(name, chunks, &dtype) }
731    }
732}
733
734impl TryFrom<(PlSmallStr, ArrayRef)> for Series {
735    type Error = PolarsError;
736
737    fn try_from(name_arr: (PlSmallStr, ArrayRef)) -> PolarsResult<Self> {
738        let (name, arr) = name_arr;
739        Series::try_from((name, vec![arr]))
740    }
741}
742
743impl TryFrom<(&ArrowField, Vec<ArrayRef>)> for Series {
744    type Error = PolarsError;
745
746    fn try_from(field_arr: (&ArrowField, Vec<ArrayRef>)) -> PolarsResult<Self> {
747        let (field, chunks) = field_arr;
748
749        let dtype = check_types(&chunks)?;
750
751        // SAFETY:
752        // dtype is checked
753        unsafe {
754            Series::_try_from_arrow_unchecked_with_md(
755                field.name.clone(),
756                chunks,
757                &dtype,
758                field.metadata.as_deref(),
759            )
760        }
761    }
762}
763
764impl TryFrom<(&ArrowField, ArrayRef)> for Series {
765    type Error = PolarsError;
766
767    fn try_from(field_arr: (&ArrowField, ArrayRef)) -> PolarsResult<Self> {
768        let (field, arr) = field_arr;
769        Series::try_from((field, vec![arr]))
770    }
771}
772
773/// Used to convert a [`ChunkedArray`], `&dyn SeriesTrait` and [`Series`]
774/// into a [`Series`].
775/// # Safety
776///
777/// This trait is marked `unsafe` as the `is_series` return is used
778/// to transmute to `Series`. This must always return `false` except
779/// for `Series` structs.
780pub unsafe trait IntoSeries {
781    fn is_series() -> bool {
782        false
783    }
784
785    fn into_series(self) -> Series
786    where
787        Self: Sized;
788}
789
790impl<T> From<ChunkedArray<T>> for Series
791where
792    T: PolarsDataType,
793    ChunkedArray<T>: IntoSeries,
794{
795    fn from(ca: ChunkedArray<T>) -> Self {
796        ca.into_series()
797    }
798}
799
800#[cfg(feature = "dtype-date")]
801impl From<DateChunked> for Series {
802    fn from(a: DateChunked) -> Self {
803        a.into_series()
804    }
805}
806
807#[cfg(feature = "dtype-datetime")]
808impl From<DatetimeChunked> for Series {
809    fn from(a: DatetimeChunked) -> Self {
810        a.into_series()
811    }
812}
813
814#[cfg(feature = "dtype-duration")]
815impl From<DurationChunked> for Series {
816    fn from(a: DurationChunked) -> Self {
817        a.into_series()
818    }
819}
820
821#[cfg(feature = "dtype-time")]
822impl From<TimeChunked> for Series {
823    fn from(a: TimeChunked) -> Self {
824        a.into_series()
825    }
826}
827
828unsafe impl IntoSeries for Arc<dyn SeriesTrait> {
829    fn into_series(self) -> Series {
830        Series(self)
831    }
832}
833
834unsafe impl IntoSeries for Series {
835    fn is_series() -> bool {
836        true
837    }
838
839    fn into_series(self) -> Series {
840        self
841    }
842}
843
844fn new_null(name: PlSmallStr, chunks: &[ArrayRef]) -> Series {
845    let len = chunks.iter().map(|arr| arr.len()).sum();
846    Series::new_null(name, len)
847}
polars_core/series/from.rs

polars_core/series/
from.rs
