Struct polars_core::prelude::CategoricalChunked

source · 
pub struct CategoricalChunked { /* private fields */ }

Implementations§

Examples found in repository?
src/series/implementations/categorical.rs (line 299)
298
299
300

    fn sort_with(&self, options: SortOptions) -> Series {
        self.0.sort_with(options).into_series()
    }
More examples
Hide additional examples
src/chunked_array/ops/sort/categorical.rs (lines 94-97)
93
94
95
96
97
98

    pub fn sort(&self, reverse: bool) -> CategoricalChunked {
        self.sort_with(SortOptions {
            nulls_last: false,
            descending: reverse,
        })
    }

Returned a sorted ChunkedArray.

Retrieve the indexes needed to sort this array.

Examples found in repository?
src/series/implementations/categorical.rs (line 303)
302
303
304

    fn argsort(&self, options: SortOptions) -> IdxCa {
        self.0.argsort(options)
    }
Examples found in repository?
src/series/implementations/categorical.rs (line 184)
182
183
184
185
186
187
188
189
190

    fn append(&mut self, other: &Series) -> PolarsResult<()> {
        if self.0.dtype() == other.dtype() {
            self.0.append(other.categorical().unwrap())
        } else {
            Err(PolarsError::SchemaMisMatch(
                "cannot append Series; data types don't match".into(),
            ))
        }
    }
Examples found in repository?
src/series/implementations/categorical.rs (line 360)
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414

    fn _sum_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn max_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn min_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn median_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn var_as_series(&self, _ddof: u8) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn std_as_series(&self, _ddof: u8) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn quantile_as_series(
        &self,
        _quantile: f64,
        _interpol: QuantileInterpolOptions,
    ) -> PolarsResult<Series> {
        Ok(CategoricalChunked::full_null(self.0.logical().name(), 1).into_series())
    }

    fn fmt_list(&self) -> String {
        FmtList::fmt_list(&self.0)
    }
    fn clone_inner(&self) -> Arc<dyn SeriesTrait> {
        Arc::new(SeriesWrap(Clone::clone(&self.0)))
    }

    #[cfg(feature = "is_in")]
    fn is_in(&self, other: &Series) -> PolarsResult<BooleanChunked> {
        _check_categorical_src(self.dtype(), other.dtype())?;
        self.0.logical().is_in(&other.to_physical_repr())
    }
    #[cfg(feature = "repeat_by")]
    fn repeat_by(&self, by: &IdxCa) -> ListChunked {
        let out = self.0.logical().repeat_by(by);
        let casted = out
            .cast(&DataType::List(Box::new(self.dtype().clone())))
            .unwrap();
        casted.list().unwrap().clone()
    }

    #[cfg(feature = "is_first")]
    fn is_first(&self) -> PolarsResult<BooleanChunked> {
        self.0.logical().is_first()
    }

    #[cfg(feature = "mode")]
    fn mode(&self) -> PolarsResult<Series> {
        Ok(CategoricalChunked::full_null(self.0.logical().name(), 1).into_series())
    }
More examples
Hide additional examples
src/chunked_array/cast.rs (line 65)
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

    fn cast_impl(&self, data_type: &DataType, checked: bool) -> PolarsResult<Series> {
        match data_type {
            #[cfg(feature = "dtype-categorical")]
            DataType::Categorical(_) => {
                Ok(CategoricalChunked::full_null(self.name(), self.len()).into_series())
            }
            #[cfg(feature = "dtype-struct")]
            DataType::Struct(fields) => {
                // cast to first field dtype
                let fld = &fields[0];
                let dtype = &fld.dtype;
                let name = &fld.name;
                let s = cast_impl_inner(name, &self.chunks, dtype, true)?;
                Ok(StructChunked::new_unchecked(self.name(), &[s]).into_series())
            }
            _ => cast_impl_inner(self.name(), &self.chunks, data_type, checked).map(|mut s| {
                // maintain sorted if data types remain signed
                // this may still fail with overflow?
                if ((self.dtype().is_signed() && data_type.is_signed())
                    || (self.dtype().is_unsigned() && data_type.is_unsigned()))
                    && (s.null_count() == self.null_count())
                {
                    let is_sorted = self.is_sorted2();
                    s.set_sorted(is_sorted)
                }
                s
            }),
        }
    }
src/series/ops/null.rs (line 11)
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

    pub fn full_null(name: &str, size: usize, dtype: &DataType) -> Self {
        // match the logical types and create them
        match dtype {
            DataType::List(inner_dtype) => {
                ListChunked::full_null_with_dtype(name, size, inner_dtype).into_series()
            }
            #[cfg(feature = "dtype-categorical")]
            DataType::Categorical(_) => CategoricalChunked::full_null(name, size).into_series(),
            #[cfg(feature = "dtype-date")]
            DataType::Date => Int32Chunked::full_null(name, size)
                .into_date()
                .into_series(),
            #[cfg(feature = "dtype-datetime")]
            DataType::Datetime(tu, tz) => Int64Chunked::full_null(name, size)
                .into_datetime(*tu, tz.clone())
                .into_series(),
            #[cfg(feature = "dtype-duration")]
            DataType::Duration(tu) => Int64Chunked::full_null(name, size)
                .into_duration(*tu)
                .into_series(),
            #[cfg(feature = "dtype-time")]
            DataType::Time => Int64Chunked::full_null(name, size)
                .into_time()
                .into_series(),
            #[cfg(feature = "dtype-struct")]
            DataType::Struct(fields) => {
                let fields = fields
                    .iter()
                    .map(|fld| Series::full_null(fld.name(), size, fld.data_type()))
                    .collect::<Vec<_>>();
                StructChunked::new(name, &fields).unwrap().into_series()
            }
            DataType::Null => ChunkedArray::new_null("", size).into_series(),
            _ => {
                macro_rules! primitive {
                    ($type:ty) => {{
                        ChunkedArray::<$type>::full_null(name, size).into_series()
                    }};
                }
                macro_rules! bool {
                    () => {{
                        ChunkedArray::<BooleanType>::full_null(name, size).into_series()
                    }};
                }
                macro_rules! utf8 {
                    () => {{
                        ChunkedArray::<Utf8Type>::full_null(name, size).into_series()
                    }};
                }
                #[cfg(feature = "dtype-binary")]
                macro_rules! binary {
                    () => {{
                        ChunkedArray::<BinaryType>::full_null(name, size).into_series()
                    }};
                }
                match_dtype_to_logical_apply_macro!(dtype, primitive, utf8, binary, bool)
            }
        }
    }
Examples found in repository?
src/series/implementations/categorical.rs (line 315)
314
315
316

    fn unique(&self) -> PolarsResult<Series> {
        self.0.unique().map(|ca| ca.into_series())
    }
Examples found in repository?
src/series/implementations/categorical.rs (line 319)
318
319
320

    fn n_unique(&self) -> PolarsResult<usize> {
        self.0.n_unique()
    }
Examples found in repository?
src/series/implementations/categorical.rs (line 272)
271
272
273

    fn len(&self) -> usize {
        self.0.len()
    }
More examples
Hide additional examples
src/chunked_array/logical/categorical/mod.rs (line 30)
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    pub fn len(&self) -> usize {
        self.logical.len()
    }

    pub fn name(&self) -> &str {
        self.logical.name()
    }

    /// Get a reference to the logical array (the categories).
    pub fn logical(&self) -> &UInt32Chunked {
        &self.logical
    }

    /// Get a reference to the logical array (the categories).
    pub(crate) fn logical_mut(&mut self) -> &mut UInt32Chunked {
        &mut self.logical
    }

    /// Build a categorical from an original RevMap. That means that the number of categories in the `RevMapping == self.unique().len()`.
    pub(crate) fn from_chunks_original(
        name: &str,
        chunks: Vec<ArrayRef>,
        rev_map: RevMapping,
    ) -> Self {
        let ca = UInt32Chunked::from_chunks(name, chunks);
        let mut logical = Logical::<UInt32Type, _>::new_logical::<CategoricalType>(ca);
        logical.2 = Some(DataType::Categorical(Some(Arc::new(rev_map))));
        let bit_settings = 1u8;
        Self {
            logical,
            bit_settings,
        }
    }

    pub fn set_lexical_sorted(&mut self, toggle: bool) {
        if toggle {
            self.bit_settings |= 1u8 << 1;
        } else {
            self.bit_settings &= !(1u8 << 1);
        }
    }

    pub(crate) fn use_lexical_sort(&self) -> bool {
        self.bit_settings & 1 << 1 != 0
    }

    /// Create a [`CategoricalChunked`] from an array of `idx` and an existing [`RevMapping`]:  `rev_map`.
    ///
    /// # Safety
    /// Invariant in `v < rev_map.len() for v in idx` must be hold.
    pub unsafe fn from_cats_and_rev_map_unchecked(
        idx: UInt32Chunked,
        rev_map: Arc<RevMapping>,
    ) -> Self {
        let mut logical = Logical::<UInt32Type, _>::new_logical::<CategoricalType>(idx);
        logical.2 = Some(DataType::Categorical(Some(rev_map)));
        Self {
            logical,
            bit_settings: Default::default(),
        }
    }

    /// # Safety
    /// The existing index values must be in bounds of the new [`RevMapping`].
    pub(crate) unsafe fn set_rev_map(&mut self, rev_map: Arc<RevMapping>, keep_fast_unique: bool) {
        self.logical.2 = Some(DataType::Categorical(Some(rev_map)));
        if !keep_fast_unique {
            self.set_fast_unique(false)
        }
    }

    pub(crate) fn can_fast_unique(&self) -> bool {
        self.bit_settings & 1 << 0 != 0 && self.logical.chunks.len() == 1
    }

    pub(crate) fn set_fast_unique(&mut self, can: bool) {
        if can {
            self.bit_settings |= 1u8 << 0;
        } else {
            self.bit_settings &= !(1u8 << 0);
        }
    }

    /// Get a reference to the mapping of categorical types to the string values.
    pub fn get_rev_map(&self) -> &Arc<RevMapping> {
        if let DataType::Categorical(Some(rev_map)) = &self.logical.2.as_ref().unwrap() {
            rev_map
        } else {
            panic!("implementation error")
        }
    }

    /// Create an `[Iterator]` that iterates over the `&str` values of the `[CategoricalChunked]`.
    pub fn iter_str(&self) -> CatIter<'_> {
        let iter = self.logical().into_iter();
        CatIter {
            rev: self.get_rev_map(),
            iter,
        }
    }
}

impl LogicalType for CategoricalChunked {
    fn dtype(&self) -> &DataType {
        self.logical.2.as_ref().unwrap()
    }

    fn get_any_value(&self, i: usize) -> PolarsResult<AnyValue<'_>> {
        if i < self.len() {
            Ok(unsafe { self.get_any_value_unchecked(i) })
        } else {
            Err(PolarsError::ComputeError("Index is out of bounds.".into()))
        }
    }

    unsafe fn get_any_value_unchecked(&self, i: usize) -> AnyValue<'_> {
        match self.logical.0.get_unchecked(i) {
            Some(i) => AnyValue::Categorical(i, self.get_rev_map()),
            None => AnyValue::Null,
        }
    }

    fn cast(&self, dtype: &DataType) -> PolarsResult<Series> {
        match dtype {
            DataType::Utf8 => {
                let mapping = &**self.get_rev_map();

                let mut builder =
                    Utf8ChunkedBuilder::new(self.logical.name(), self.len(), self.len() * 5);

                let f = |idx: u32| mapping.get(idx);

                if !self.logical.has_validity() {
                    self.logical
                        .into_no_null_iter()
                        .for_each(|idx| builder.append_value(f(idx)));
                } else {
                    self.logical.into_iter().for_each(|opt_idx| {
                        builder.append_option(opt_idx.map(f));
                    });
                }

                let ca = builder.finish();
                Ok(ca.into_series())
            }
            DataType::UInt32 => {
                let ca =
                    UInt32Chunked::from_chunks(self.logical.name(), self.logical.chunks.clone());
                Ok(ca.into_series())
            }
            #[cfg(feature = "dtype-categorical")]
            DataType::Categorical(_) => Ok(self.clone().into_series()),
            _ => self.logical.cast(dtype),
        }
    }
src/chunked_array/ops/sort/categorical.rs (line 109)
101
102
103
104
105
106
107
108
109
110
111
112
113
114

    pub fn argsort(&self, options: SortOptions) -> IdxCa {
        if self.use_lexical_sort() {
            let iters = [self.iter_str()];
            argsort::argsort(
                self.name(),
                iters,
                options,
                self.logical().null_count(),
                self.len(),
            )
        } else {
            self.logical().argsort(options)
        }
    }
src/series/comparison.rs (line 77)
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

fn compare_cat_to_str_value<Compare>(
    cat: &Series,
    value: &str,
    name: &str,
    compare: Compare,
    fill_value: bool,
) -> PolarsResult<BooleanChunked>
where
    Compare: Fn(&Series, u32) -> PolarsResult<BooleanChunked>,
{
    let cat = cat.categorical().expect("should be categorical");
    let cat_map = cat.get_rev_map();
    match cat_map.find(value) {
        None => Ok(BooleanChunked::full(name, fill_value, cat.len())),
        Some(cat_idx) => {
            let cat = cat.cast(&DataType::UInt32).unwrap();
            compare(&cat, cat_idx)
        }
    }
}
src/chunked_array/logical/categorical/ops/append.rs (line 7)
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

    pub fn append(&mut self, other: &Self) -> PolarsResult<()> {
        if self.logical.null_count() == self.len() && other.logical.null_count() == other.len() {
            let len = self.len();
            self.logical_mut().length += other.len() as IdxSize;
            new_chunks(&mut self.logical.chunks, &other.logical().chunks, len);
            return Ok(());
        }
        let is_local_different_source =
            match (self.get_rev_map().as_ref(), other.get_rev_map().as_ref()) {
                (RevMapping::Local(arr_l), RevMapping::Local(arr_r)) => !std::ptr::eq(arr_l, arr_r),
                _ => false,
            };

        if is_local_different_source {
            return Err(PolarsError::ComputeError("Cannot concat Categoricals coming from a different source. Consider setting a global StringCache.".into()));
        } else {
            let len = self.len();
            let new_rev_map = self.merge_categorical_map(other)?;
            unsafe { self.set_rev_map(new_rev_map, false) };

            self.logical_mut().length += other.len() as IdxSize;
            new_chunks(&mut self.logical.chunks, &other.logical().chunks, len);
        }
        self.logical.set_sorted2(IsSorted::Not);
        Ok(())
    }
Examples found in repository?
src/chunked_array/ops/sort/categorical.rs (line 105)
101
102
103
104
105
106
107
108
109
110
111
112
113
114

    pub fn argsort(&self, options: SortOptions) -> IdxCa {
        if self.use_lexical_sort() {
            let iters = [self.iter_str()];
            argsort::argsort(
                self.name(),
                iters,
                options,
                self.logical().null_count(),
                self.len(),
            )
        } else {
            self.logical().argsort(options)
        }
    }

Get a reference to the logical array (the categories).

Examples found in repository?
src/chunked_array/logical/categorical/mod.rs (line 25)
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

    pub(crate) fn field(&self) -> Field {
        let name = self.logical().name();
        Field::new(name, self.dtype().clone())
    }

    pub fn is_empty(&self) -> bool {
        self.len() == 0
    }

    pub fn len(&self) -> usize {
        self.logical.len()
    }

    pub fn name(&self) -> &str {
        self.logical.name()
    }

    /// Get a reference to the logical array (the categories).
    pub fn logical(&self) -> &UInt32Chunked {
        &self.logical
    }

    /// Get a reference to the logical array (the categories).
    pub(crate) fn logical_mut(&mut self) -> &mut UInt32Chunked {
        &mut self.logical
    }

    /// Build a categorical from an original RevMap. That means that the number of categories in the `RevMapping == self.unique().len()`.
    pub(crate) fn from_chunks_original(
        name: &str,
        chunks: Vec<ArrayRef>,
        rev_map: RevMapping,
    ) -> Self {
        let ca = UInt32Chunked::from_chunks(name, chunks);
        let mut logical = Logical::<UInt32Type, _>::new_logical::<CategoricalType>(ca);
        logical.2 = Some(DataType::Categorical(Some(Arc::new(rev_map))));
        let bit_settings = 1u8;
        Self {
            logical,
            bit_settings,
        }
    }

    pub fn set_lexical_sorted(&mut self, toggle: bool) {
        if toggle {
            self.bit_settings |= 1u8 << 1;
        } else {
            self.bit_settings &= !(1u8 << 1);
        }
    }

    pub(crate) fn use_lexical_sort(&self) -> bool {
        self.bit_settings & 1 << 1 != 0
    }

    /// Create a [`CategoricalChunked`] from an array of `idx` and an existing [`RevMapping`]:  `rev_map`.
    ///
    /// # Safety
    /// Invariant in `v < rev_map.len() for v in idx` must be hold.
    pub unsafe fn from_cats_and_rev_map_unchecked(
        idx: UInt32Chunked,
        rev_map: Arc<RevMapping>,
    ) -> Self {
        let mut logical = Logical::<UInt32Type, _>::new_logical::<CategoricalType>(idx);
        logical.2 = Some(DataType::Categorical(Some(rev_map)));
        Self {
            logical,
            bit_settings: Default::default(),
        }
    }

    /// # Safety
    /// The existing index values must be in bounds of the new [`RevMapping`].
    pub(crate) unsafe fn set_rev_map(&mut self, rev_map: Arc<RevMapping>, keep_fast_unique: bool) {
        self.logical.2 = Some(DataType::Categorical(Some(rev_map)));
        if !keep_fast_unique {
            self.set_fast_unique(false)
        }
    }

    pub(crate) fn can_fast_unique(&self) -> bool {
        self.bit_settings & 1 << 0 != 0 && self.logical.chunks.len() == 1
    }

    pub(crate) fn set_fast_unique(&mut self, can: bool) {
        if can {
            self.bit_settings |= 1u8 << 0;
        } else {
            self.bit_settings &= !(1u8 << 0);
        }
    }

    /// Get a reference to the mapping of categorical types to the string values.
    pub fn get_rev_map(&self) -> &Arc<RevMapping> {
        if let DataType::Categorical(Some(rev_map)) = &self.logical.2.as_ref().unwrap() {
            rev_map
        } else {
            panic!("implementation error")
        }
    }

    /// Create an `[Iterator]` that iterates over the `&str` values of the `[CategoricalChunked]`.
    pub fn iter_str(&self) -> CatIter<'_> {
        let iter = self.logical().into_iter();
        CatIter {
            rev: self.get_rev_map(),
            iter,
        }
    }
More examples
Hide additional examples
src/series/implementations/categorical.rs (line 41)
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423

    fn with_state<F>(&self, keep_fast_unique: bool, apply: F) -> CategoricalChunked
    where
        F: Fn(&UInt32Chunked) -> UInt32Chunked,
    {
        let cats = apply(self.0.logical());
        self.finish_with_state(keep_fast_unique, cats)
    }

    fn try_with_state<'a, F>(
        &'a self,
        keep_fast_unique: bool,
        apply: F,
    ) -> PolarsResult<CategoricalChunked>
    where
        F: for<'b> Fn(&'a UInt32Chunked) -> PolarsResult<UInt32Chunked>,
    {
        let cats = apply(self.0.logical())?;
        Ok(self.finish_with_state(keep_fast_unique, cats))
    }
}

impl private::PrivateSeries for SeriesWrap<CategoricalChunked> {
    fn compute_len(&mut self) {
        self.0.logical_mut().compute_len()
    }
    fn _field(&self) -> Cow<Field> {
        Cow::Owned(self.0.field())
    }
    fn _dtype(&self) -> &DataType {
        self.0.dtype()
    }

    fn explode_by_offsets(&self, offsets: &[i64]) -> Series {
        // TODO! explode by offset should return concrete type
        self.with_state(true, |cats| {
            cats.explode_by_offsets(offsets).u32().unwrap().clone()
        })
        .into_series()
    }

    fn _set_sorted(&mut self, is_sorted: IsSorted) {
        self.0.logical_mut().set_sorted2(is_sorted)
    }

    unsafe fn equal_element(&self, idx_self: usize, idx_other: usize, other: &Series) -> bool {
        self.0.logical().equal_element(idx_self, idx_other, other)
    }

    #[cfg(feature = "zip_with")]
    fn zip_with_same_type(&self, mask: &BooleanChunked, other: &Series) -> PolarsResult<Series> {
        self.0
            .zip_with(mask, other.categorical()?)
            .map(|ca| ca.into_series())
    }
    fn into_partial_ord_inner<'a>(&'a self) -> Box<dyn PartialOrdInner + 'a> {
        (&self.0).into_partial_ord_inner()
    }

    fn vec_hash(&self, random_state: RandomState, buf: &mut Vec<u64>) -> PolarsResult<()> {
        self.0.logical().vec_hash(random_state, buf);
        Ok(())
    }

    fn vec_hash_combine(&self, build_hasher: RandomState, hashes: &mut [u64]) -> PolarsResult<()> {
        self.0.logical().vec_hash_combine(build_hasher, hashes);
        Ok(())
    }

    unsafe fn agg_list(&self, groups: &GroupsProxy) -> Series {
        // we cannot cast and dispatch as the inner type of the list would be incorrect
        self.0
            .logical()
            .agg_list(groups)
            .cast(&DataType::List(Box::new(self.dtype().clone())))
            .unwrap()
    }

    fn zip_outer_join_column(
        &self,
        right_column: &Series,
        opt_join_tuples: &[(Option<IdxSize>, Option<IdxSize>)],
    ) -> Series {
        let new_rev_map = self
            .0
            .merge_categorical_map(right_column.categorical().unwrap())
            .unwrap();
        let left = self.0.logical();
        let right = right_column
            .categorical()
            .unwrap()
            .logical()
            .clone()
            .into_series();

        let cats = left.zip_outer_join_column(&right, opt_join_tuples);
        let cats = cats.u32().unwrap().clone();

        unsafe {
            CategoricalChunked::from_cats_and_rev_map_unchecked(cats, new_rev_map).into_series()
        }
    }
    fn group_tuples(&self, multithreaded: bool, sorted: bool) -> PolarsResult<GroupsProxy> {
        self.0.logical().group_tuples(multithreaded, sorted)
    }

    #[cfg(feature = "sort_multiple")]
    fn argsort_multiple(&self, by: &[Series], reverse: &[bool]) -> PolarsResult<IdxCa> {
        self.0.argsort_multiple(by, reverse)
    }
}

impl SeriesTrait for SeriesWrap<CategoricalChunked> {
    fn is_sorted(&self) -> IsSorted {
        if self.0.logical().is_sorted() {
            IsSorted::Ascending
        } else if self.0.logical().is_sorted_reverse() {
            IsSorted::Descending
        } else {
            IsSorted::Not
        }
    }

    fn rename(&mut self, name: &str) {
        self.0.logical_mut().rename(name);
    }

    fn chunk_lengths(&self) -> ChunkIdIter {
        self.0.logical().chunk_id()
    }
    fn name(&self) -> &str {
        self.0.logical().name()
    }

    fn chunks(&self) -> &Vec<ArrayRef> {
        self.0.logical().chunks()
    }
    fn shrink_to_fit(&mut self) {
        self.0.logical_mut().shrink_to_fit()
    }

    fn slice(&self, offset: i64, length: usize) -> Series {
        self.with_state(false, |cats| cats.slice(offset, length))
            .into_series()
    }

    fn append(&mut self, other: &Series) -> PolarsResult<()> {
        if self.0.dtype() == other.dtype() {
            self.0.append(other.categorical().unwrap())
        } else {
            Err(PolarsError::SchemaMisMatch(
                "cannot append Series; data types don't match".into(),
            ))
        }
    }
    fn extend(&mut self, other: &Series) -> PolarsResult<()> {
        if self.0.dtype() == other.dtype() {
            let other = other.categorical()?;
            self.0.logical_mut().extend(other.logical());
            let new_rev_map = self.0.merge_categorical_map(other)?;
            // safety:
            // rev_maps are merged
            unsafe { self.0.set_rev_map(new_rev_map, false) };
            Ok(())
        } else {
            Err(PolarsError::SchemaMisMatch(
                "cannot extend Series; data types don't match".into(),
            ))
        }
    }

    fn filter(&self, filter: &BooleanChunked) -> PolarsResult<Series> {
        self.try_with_state(false, |cats| cats.filter(filter))
            .map(|ca| ca.into_series())
    }

    #[cfg(feature = "chunked_ids")]
    unsafe fn _take_chunked_unchecked(&self, by: &[ChunkId], sorted: IsSorted) -> Series {
        let cats = self.0.logical().take_chunked_unchecked(by, sorted);
        self.finish_with_state(false, cats).into_series()
    }

    #[cfg(feature = "chunked_ids")]
    unsafe fn _take_opt_chunked_unchecked(&self, by: &[Option<ChunkId>]) -> Series {
        let cats = self.0.logical().take_opt_chunked_unchecked(by);
        self.finish_with_state(false, cats).into_series()
    }

    fn take(&self, indices: &IdxCa) -> PolarsResult<Series> {
        let indices = if indices.chunks.len() > 1 {
            Cow::Owned(indices.rechunk())
        } else {
            Cow::Borrowed(indices)
        };
        self.try_with_state(false, |cats| cats.take((&*indices).into()))
            .map(|ca| ca.into_series())
    }

    fn take_iter(&self, iter: &mut dyn TakeIterator) -> PolarsResult<Series> {
        let cats = self.0.logical().take(iter.into())?;
        Ok(self.finish_with_state(false, cats).into_series())
    }

    fn take_every(&self, n: usize) -> Series {
        self.with_state(true, |cats| cats.take_every(n))
            .into_series()
    }

    unsafe fn take_iter_unchecked(&self, iter: &mut dyn TakeIterator) -> Series {
        let cats = self.0.logical().take_unchecked(iter.into());
        self.finish_with_state(false, cats).into_series()
    }

    unsafe fn take_unchecked(&self, idx: &IdxCa) -> PolarsResult<Series> {
        let idx = if idx.chunks.len() > 1 {
            Cow::Owned(idx.rechunk())
        } else {
            Cow::Borrowed(idx)
        };
        Ok(self
            .with_state(false, |cats| cats.take_unchecked((&*idx).into()))
            .into_series())
    }

    unsafe fn take_opt_iter_unchecked(&self, iter: &mut dyn TakeIteratorNulls) -> Series {
        let cats = self.0.logical().take_unchecked(iter.into());
        self.finish_with_state(false, cats).into_series()
    }

    #[cfg(feature = "take_opt_iter")]
    fn take_opt_iter(&self, iter: &mut dyn TakeIteratorNulls) -> PolarsResult<Series> {
        let cats = self.0.logical().take(iter.into())?;
        Ok(self.finish_with_state(false, cats).into_series())
    }

    fn len(&self) -> usize {
        self.0.len()
    }

    fn rechunk(&self) -> Series {
        self.with_state(true, |ca| ca.rechunk()).into_series()
    }

    fn new_from_index(&self, index: usize, length: usize) -> Series {
        self.with_state(true, |cats| cats.new_from_index(index, length))
            .into_series()
    }

    fn cast(&self, data_type: &DataType) -> PolarsResult<Series> {
        self.0.cast(data_type)
    }

    fn get(&self, index: usize) -> PolarsResult<AnyValue> {
        self.0.get_any_value(index)
    }

    #[inline]
    #[cfg(feature = "private")]
    unsafe fn get_unchecked(&self, index: usize) -> AnyValue {
        self.0.get_any_value_unchecked(index)
    }

    fn sort_with(&self, options: SortOptions) -> Series {
        self.0.sort_with(options).into_series()
    }

    fn argsort(&self, options: SortOptions) -> IdxCa {
        self.0.argsort(options)
    }

    fn null_count(&self) -> usize {
        self.0.logical().null_count()
    }

    fn has_validity(&self) -> bool {
        self.0.logical().has_validity()
    }

    fn unique(&self) -> PolarsResult<Series> {
        self.0.unique().map(|ca| ca.into_series())
    }

    fn n_unique(&self) -> PolarsResult<usize> {
        self.0.n_unique()
    }

    fn arg_unique(&self) -> PolarsResult<IdxCa> {
        self.0.logical().arg_unique()
    }

    fn is_null(&self) -> BooleanChunked {
        self.0.logical().is_null()
    }

    fn is_not_null(&self) -> BooleanChunked {
        self.0.logical().is_not_null()
    }

    fn is_unique(&self) -> PolarsResult<BooleanChunked> {
        self.0.logical().is_unique()
    }

    fn is_duplicated(&self) -> PolarsResult<BooleanChunked> {
        self.0.logical().is_duplicated()
    }

    fn reverse(&self) -> Series {
        self.with_state(true, |cats| cats.reverse()).into_series()
    }

    fn as_single_ptr(&mut self) -> PolarsResult<usize> {
        self.0.logical_mut().as_single_ptr()
    }

    fn shift(&self, periods: i64) -> Series {
        self.with_state(false, |ca| ca.shift(periods)).into_series()
    }

    fn fill_null(&self, strategy: FillNullStrategy) -> PolarsResult<Series> {
        self.try_with_state(false, |cats| cats.fill_null(strategy))
            .map(|ca| ca.into_series())
    }

    fn _sum_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn max_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn min_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn median_as_series(&self) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn var_as_series(&self, _ddof: u8) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn std_as_series(&self, _ddof: u8) -> Series {
        CategoricalChunked::full_null(self.0.logical().name(), 1).into_series()
    }
    fn quantile_as_series(
        &self,
        _quantile: f64,
        _interpol: QuantileInterpolOptions,
    ) -> PolarsResult<Series> {
        Ok(CategoricalChunked::full_null(self.0.logical().name(), 1).into_series())
    }

    fn fmt_list(&self) -> String {
        FmtList::fmt_list(&self.0)
    }
    fn clone_inner(&self) -> Arc<dyn SeriesTrait> {
        Arc::new(SeriesWrap(Clone::clone(&self.0)))
    }

    #[cfg(feature = "is_in")]
    fn is_in(&self, other: &Series) -> PolarsResult<BooleanChunked> {
        _check_categorical_src(self.dtype(), other.dtype())?;
        self.0.logical().is_in(&other.to_physical_repr())
    }
    #[cfg(feature = "repeat_by")]
    fn repeat_by(&self, by: &IdxCa) -> ListChunked {
        let out = self.0.logical().repeat_by(by);
        let casted = out
            .cast(&DataType::List(Box::new(self.dtype().clone())))
            .unwrap();
        casted.list().unwrap().clone()
    }

    #[cfg(feature = "is_first")]
    fn is_first(&self) -> PolarsResult<BooleanChunked> {
        self.0.logical().is_first()
    }

    #[cfg(feature = "mode")]
    fn mode(&self) -> PolarsResult<Series> {
        Ok(CategoricalChunked::full_null(self.0.logical().name(), 1).into_series())
    }
}

impl private::PrivateSeriesNumeric for SeriesWrap<CategoricalChunked> {
    fn bit_repr_is_large(&self) -> bool {
        false
    }
    fn bit_repr_small(&self) -> UInt32Chunked {
        self.0.logical().clone()
    }
src/chunked_array/logical/categorical/ops/take_random.rs (line 27)
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

    pub(crate) fn new(ca: &'a CategoricalChunked) -> Self {
        // should be rechunked upstream
        assert_eq!(ca.logical.chunks.len(), 1, "implementation error");
        if let RevMapping::Local(rev_map) = &**ca.get_rev_map() {
            let cats = ca.logical().take_rand();
            Self { rev_map, cats }
        } else {
            unreachable!()
        }
    }
}

impl PartialOrdInner for CategoricalTakeRandomLocal<'_> {
    unsafe fn cmp_element_unchecked(&self, idx_a: usize, idx_b: usize) -> Ordering {
        let a = self
            .cats
            .get_unchecked(idx_a)
            .map(|cat| self.rev_map.value_unchecked(cat as usize));
        let b = self
            .cats
            .get_unchecked(idx_b)
            .map(|cat| self.rev_map.value_unchecked(cat as usize));
        a.partial_cmp(&b).unwrap()
    }
}

pub(crate) struct CategoricalTakeRandomGlobal<'a> {
    rev_map_part_1: &'a PlHashMap<u32, u32>,
    rev_map_part_2: &'a Utf8Array<i64>,
    cats: TakeCats<'a>,
}
impl<'a> CategoricalTakeRandomGlobal<'a> {
    pub(crate) fn new(ca: &'a CategoricalChunked) -> Self {
        // should be rechunked upstream
        assert_eq!(ca.logical.chunks.len(), 1, "implementation error");
        if let RevMapping::Global(rev_map_part_1, rev_map_part_2, _) = &**ca.get_rev_map() {
            let cats = ca.logical().take_rand();
            Self {
                rev_map_part_1,
                rev_map_part_2,
                cats,
            }
        } else {
            unreachable!()
        }
    }
src/chunked_array/logical/categorical/ops/zip.rs (line 13)
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

    pub(crate) fn zip_with(
        &self,
        mask: &BooleanChunked,
        other: &CategoricalChunked,
    ) -> PolarsResult<Self> {
        let cats = match &**self.get_rev_map() {
            RevMapping::Local(rev_map) => {
                // the logic for merging the rev maps will concatenate utf8 arrays
                // to make sure the indexes still make sense we need to offset the right hand side
                self.logical()
                    .zip_with(mask, &(other.logical() + rev_map.len() as u32))?
            }
            _ => self.logical().zip_with(mask, other.logical())?,
        };
        let new_state = self.merge_categorical_map(other)?;

        // Safety:
        // we checked the rev_maps.
        unsafe {
            Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
                cats, new_state,
            ))
        }
    }
src/chunked_array/logical/categorical/ops/unique.rs (line 10)
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

    pub fn unique(&self) -> PolarsResult<Self> {
        let cat_map = self.get_rev_map();
        if self.can_fast_unique() {
            let ca = match &**cat_map {
                RevMapping::Local(a) => {
                    UInt32Chunked::from_iter_values(self.logical().name(), 0..(a.len() as u32))
                }
                RevMapping::Global(map, _, _) => {
                    UInt32Chunked::from_iter_values(self.logical().name(), map.keys().copied())
                }
            };
            // safety:
            // we only removed some indexes so we are still in bounds
            unsafe {
                let mut out =
                    CategoricalChunked::from_cats_and_rev_map_unchecked(ca, cat_map.clone());
                out.set_fast_unique(true);
                Ok(out)
            }
        } else {
            let ca = self.logical().unique()?;
            // safety:
            // we only removed some indexes so we are still in bounds
            unsafe {
                Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
                    ca,
                    cat_map.clone(),
                ))
            }
        }
    }

    pub fn n_unique(&self) -> PolarsResult<usize> {
        if self.can_fast_unique() {
            Ok(self.get_rev_map().len())
        } else {
            self.logical().n_unique()
        }
    }

    pub fn value_counts(&self) -> PolarsResult<DataFrame> {
        let groups = self.logical().group_tuples(true, false).unwrap();
        let logical_values = unsafe {
            self.logical()
                .clone()
                .into_series()
                .agg_first(&groups)
                .u32()
                .unwrap()
                .clone()
        };

        let mut values = self.clone();
        *values.logical_mut() = logical_values;

        let mut counts = groups.group_count();
        counts.rename("counts");
        let cols = vec![values.into_series(), counts.into_series()];
        let df = DataFrame::new_no_checks(cols);
        df.sort(["counts"], true)
    }
src/chunked_array/logical/categorical/from.rs (line 10)
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

    fn from(ca: &CategoricalChunked) -> Self {
        let keys = ca.logical().rechunk();
        let keys = keys.downcast_iter().next().unwrap();
        let map = &**ca.get_rev_map();
        let dtype = ArrowDataType::Dictionary(
            IntegerType::UInt32,
            Box::new(ArrowDataType::LargeUtf8),
            false,
        );
        match map {
            RevMapping::Local(arr) => {
                // Safety:
                // the keys are in bounds
                unsafe {
                    DictionaryArray::try_new_unchecked(dtype, keys.clone(), Box::new(arr.clone()))
                        .unwrap()
                }
            }
            RevMapping::Global(reverse_map, values, _uuid) => {
                let iter = keys
                    .into_iter()
                    .map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap()));
                let keys = PrimitiveArray::from_trusted_len_iter(iter);

                // Safety:
                // the keys are in bounds
                unsafe {
                    DictionaryArray::try_new_unchecked(dtype, keys, Box::new(values.clone()))
                        .unwrap()
                }
            }
        }
    }
}
impl From<&CategoricalChunked> for DictionaryArray<i64> {
    fn from(ca: &CategoricalChunked) -> Self {
        let keys = ca.logical().rechunk();
        let keys = keys.downcast_iter().next().unwrap();
        let map = &**ca.get_rev_map();
        let dtype = ArrowDataType::Dictionary(
            IntegerType::UInt32,
            Box::new(ArrowDataType::LargeUtf8),
            false,
        );
        match map {
            // Safety:
            // the keys are in bounds
            RevMapping::Local(arr) => unsafe {
                DictionaryArray::try_new_unchecked(
                    dtype,
                    cast(keys, &ArrowDataType::Int64)
                        .unwrap()
                        .as_any()
                        .downcast_ref::<PrimitiveArray<i64>>()
                        .unwrap()
                        .clone(),
                    Box::new(arr.clone()),
                )
                .unwrap()
            },
            RevMapping::Global(reverse_map, values, _uuid) => {
                let iter = keys
                    .into_iter()
                    .map(|opt_k| opt_k.map(|k| *reverse_map.get(k).unwrap() as i64));
                let keys = PrimitiveArray::from_trusted_len_iter(iter);

                // Safety:
                // the keys are in bounds
                unsafe {
                    DictionaryArray::try_new_unchecked(dtype, keys, Box::new(values.clone()))
                        .unwrap()
                }
            }
        }
    }
Additional examples can be found in:
Examples found in repository?
src/series/implementations/categorical.rs (line 33)
26
27
28
29
30
31
32
33
34
35

    fn finish_with_state(&self, keep_fast_unique: bool, cats: UInt32Chunked) -> CategoricalChunked {
        let mut out = unsafe {
            CategoricalChunked::from_cats_and_rev_map_unchecked(cats, self.0.get_rev_map().clone())
        };
        if keep_fast_unique && self.0.can_fast_unique() {
            out.set_fast_unique(true)
        }
        out.set_lexical_sorted(self.0.use_lexical_sort());
        out
    }

Create a CategoricalChunked from an array of idx and an existing RevMapping: rev_map.

Safety

Invariant in v < rev_map.len() for v in idx must be hold.

Examples found in repository?
src/chunked_array/logical/categorical/ops/full.rs (lines 8-11)
4
5
6
7
8
9
10
11
12
13

    pub fn full_null(name: &str, length: usize) -> CategoricalChunked {
        let cats = UInt32Chunked::full_null(name, length);

        unsafe {
            CategoricalChunked::from_cats_and_rev_map_unchecked(
                cats,
                Arc::new(RevMapping::default()),
            )
        }
    }
More examples
Hide additional examples
src/series/implementations/categorical.rs (line 28)
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

    fn finish_with_state(&self, keep_fast_unique: bool, cats: UInt32Chunked) -> CategoricalChunked {
        let mut out = unsafe {
            CategoricalChunked::from_cats_and_rev_map_unchecked(cats, self.0.get_rev_map().clone())
        };
        if keep_fast_unique && self.0.can_fast_unique() {
            out.set_fast_unique(true)
        }
        out.set_lexical_sorted(self.0.use_lexical_sort());
        out
    }

    fn with_state<F>(&self, keep_fast_unique: bool, apply: F) -> CategoricalChunked
    where
        F: Fn(&UInt32Chunked) -> UInt32Chunked,
    {
        let cats = apply(self.0.logical());
        self.finish_with_state(keep_fast_unique, cats)
    }

    fn try_with_state<'a, F>(
        &'a self,
        keep_fast_unique: bool,
        apply: F,
    ) -> PolarsResult<CategoricalChunked>
    where
        F: for<'b> Fn(&'a UInt32Chunked) -> PolarsResult<UInt32Chunked>,
    {
        let cats = apply(self.0.logical())?;
        Ok(self.finish_with_state(keep_fast_unique, cats))
    }
}

impl private::PrivateSeries for SeriesWrap<CategoricalChunked> {
    fn compute_len(&mut self) {
        self.0.logical_mut().compute_len()
    }
    fn _field(&self) -> Cow<Field> {
        Cow::Owned(self.0.field())
    }
    fn _dtype(&self) -> &DataType {
        self.0.dtype()
    }

    fn explode_by_offsets(&self, offsets: &[i64]) -> Series {
        // TODO! explode by offset should return concrete type
        self.with_state(true, |cats| {
            cats.explode_by_offsets(offsets).u32().unwrap().clone()
        })
        .into_series()
    }

    fn _set_sorted(&mut self, is_sorted: IsSorted) {
        self.0.logical_mut().set_sorted2(is_sorted)
    }

    unsafe fn equal_element(&self, idx_self: usize, idx_other: usize, other: &Series) -> bool {
        self.0.logical().equal_element(idx_self, idx_other, other)
    }

    #[cfg(feature = "zip_with")]
    fn zip_with_same_type(&self, mask: &BooleanChunked, other: &Series) -> PolarsResult<Series> {
        self.0
            .zip_with(mask, other.categorical()?)
            .map(|ca| ca.into_series())
    }
    fn into_partial_ord_inner<'a>(&'a self) -> Box<dyn PartialOrdInner + 'a> {
        (&self.0).into_partial_ord_inner()
    }

    fn vec_hash(&self, random_state: RandomState, buf: &mut Vec<u64>) -> PolarsResult<()> {
        self.0.logical().vec_hash(random_state, buf);
        Ok(())
    }

    fn vec_hash_combine(&self, build_hasher: RandomState, hashes: &mut [u64]) -> PolarsResult<()> {
        self.0.logical().vec_hash_combine(build_hasher, hashes);
        Ok(())
    }

    unsafe fn agg_list(&self, groups: &GroupsProxy) -> Series {
        // we cannot cast and dispatch as the inner type of the list would be incorrect
        self.0
            .logical()
            .agg_list(groups)
            .cast(&DataType::List(Box::new(self.dtype().clone())))
            .unwrap()
    }

    fn zip_outer_join_column(
        &self,
        right_column: &Series,
        opt_join_tuples: &[(Option<IdxSize>, Option<IdxSize>)],
    ) -> Series {
        let new_rev_map = self
            .0
            .merge_categorical_map(right_column.categorical().unwrap())
            .unwrap();
        let left = self.0.logical();
        let right = right_column
            .categorical()
            .unwrap()
            .logical()
            .clone()
            .into_series();

        let cats = left.zip_outer_join_column(&right, opt_join_tuples);
        let cats = cats.u32().unwrap().clone();

        unsafe {
            CategoricalChunked::from_cats_and_rev_map_unchecked(cats, new_rev_map).into_series()
        }
    }
src/chunked_array/logical/categorical/ops/zip.rs (lines 23-25)
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

    pub(crate) fn zip_with(
        &self,
        mask: &BooleanChunked,
        other: &CategoricalChunked,
    ) -> PolarsResult<Self> {
        let cats = match &**self.get_rev_map() {
            RevMapping::Local(rev_map) => {
                // the logic for merging the rev maps will concatenate utf8 arrays
                // to make sure the indexes still make sense we need to offset the right hand side
                self.logical()
                    .zip_with(mask, &(other.logical() + rev_map.len() as u32))?
            }
            _ => self.logical().zip_with(mask, other.logical())?,
        };
        let new_state = self.merge_categorical_map(other)?;

        // Safety:
        // we checked the rev_maps.
        unsafe {
            Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
                cats, new_state,
            ))
        }
    }
src/chunked_array/logical/categorical/ops/unique.rs (line 20)
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

    pub fn unique(&self) -> PolarsResult<Self> {
        let cat_map = self.get_rev_map();
        if self.can_fast_unique() {
            let ca = match &**cat_map {
                RevMapping::Local(a) => {
                    UInt32Chunked::from_iter_values(self.logical().name(), 0..(a.len() as u32))
                }
                RevMapping::Global(map, _, _) => {
                    UInt32Chunked::from_iter_values(self.logical().name(), map.keys().copied())
                }
            };
            // safety:
            // we only removed some indexes so we are still in bounds
            unsafe {
                let mut out =
                    CategoricalChunked::from_cats_and_rev_map_unchecked(ca, cat_map.clone());
                out.set_fast_unique(true);
                Ok(out)
            }
        } else {
            let ca = self.logical().unique()?;
            // safety:
            // we only removed some indexes so we are still in bounds
            unsafe {
                Ok(CategoricalChunked::from_cats_and_rev_map_unchecked(
                    ca,
                    cat_map.clone(),
                ))
            }
        }
    }
src/series/into.rs (lines 53-56)
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76

    pub fn to_arrow(&self, chunk_idx: usize) -> ArrayRef {
        match self.dtype() {
            // special list branch to
            // make sure that we recursively apply all logical types.
            DataType::List(inner) => {
                let ca = self.list().unwrap();
                let arr = ca.chunks[chunk_idx].clone();
                let arr = arr.as_any().downcast_ref::<ListArray<i64>>().unwrap();

                let s = unsafe {
                    Series::from_chunks_and_dtype_unchecked("", vec![arr.values().clone()], inner)
                };
                let new_values = s.to_arrow(0);

                let data_type = ListArray::<i64>::default_datatype(inner.to_arrow());
                let arr = ListArray::<i64>::new(
                    data_type,
                    arr.offsets().clone(),
                    new_values,
                    arr.validity().cloned(),
                );
                Box::new(arr)
            }
            #[cfg(feature = "dtype-categorical")]
            DataType::Categorical(_) => {
                let ca = self.categorical().unwrap();
                let arr = ca.logical().chunks()[chunk_idx].clone();
                let cats = UInt32Chunked::from_chunks("", vec![arr]);

                // safety:
                // we only take a single chunk and change nothing about the index/rev_map mapping
                let new = unsafe {
                    CategoricalChunked::from_cats_and_rev_map_unchecked(
                        cats,
                        ca.get_rev_map().clone(),
                    )
                };

                let arr: DictionaryArray<u32> = (&new).into();
                Box::new(arr) as ArrayRef
            }
            #[cfg(feature = "dtype-date")]
            DataType::Date => cast(&*self.chunks()[chunk_idx], &DataType::Date.to_arrow()).unwrap(),
            #[cfg(feature = "dtype-datetime")]
            DataType::Datetime(_, _) => {
                cast(&*self.chunks()[chunk_idx], &self.dtype().to_arrow()).unwrap()
            }
            #[cfg(feature = "dtype-duration")]
            DataType::Duration(_) => {
                cast(&*self.chunks()[chunk_idx], &self.dtype().to_arrow()).unwrap()
            }
            #[cfg(feature = "dtype-time")]
            DataType::Time => cast(&*self.chunks()[chunk_idx], &DataType::Time.to_arrow()).unwrap(),
            _ => self.array_ref(chunk_idx).clone(),
        }
    }
src/frame/hash_join/mod.rs (line 568)
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581

    pub fn _outer_join_from_series(
        &self,
        other: &DataFrame,
        s_left: &Series,
        s_right: &Series,
        suffix: Option<String>,
        slice: Option<(i64, usize)>,
    ) -> PolarsResult<DataFrame> {
        #[cfg(feature = "dtype-categorical")]
        _check_categorical_src(s_left.dtype(), s_right.dtype())?;

        // store this so that we can keep original column order.
        let join_column_index = self.iter().position(|s| s.name() == s_left.name()).unwrap();

        // Get the indexes of the joined relations
        let opt_join_tuples = s_left.hash_join_outer(s_right);
        let mut opt_join_tuples = &*opt_join_tuples;

        if let Some((offset, len)) = slice {
            opt_join_tuples = slice_slice(opt_join_tuples, offset, len);
        }

        // Take the left and right dataframes by join tuples
        let (mut df_left, df_right) = POOL.join(
            || unsafe {
                self.drop(s_left.name()).unwrap().take_opt_iter_unchecked(
                    opt_join_tuples
                        .iter()
                        .map(|(left, _right)| left.map(|i| i as usize)),
                )
            },
            || unsafe {
                other.drop(s_right.name()).unwrap().take_opt_iter_unchecked(
                    opt_join_tuples
                        .iter()
                        .map(|(_left, right)| right.map(|i| i as usize)),
                )
            },
        );

        let mut s = s_left
            .to_physical_repr()
            .zip_outer_join_column(&s_right.to_physical_repr(), opt_join_tuples);
        s.rename(s_left.name());
        let s = match s_left.dtype() {
            #[cfg(feature = "dtype-categorical")]
            DataType::Categorical(_) => {
                let ca_left = s_left.categorical().unwrap();
                let new_rev_map = ca_left.merge_categorical_map(s_right.categorical().unwrap())?;
                let logical = s.u32().unwrap().clone();
                // safety:
                // categorical maps are merged
                unsafe {
                    CategoricalChunked::from_cats_and_rev_map_unchecked(logical, new_rev_map)
                        .into_series()
                }
            }
            dt @ DataType::Datetime(_, _)
            | dt @ DataType::Time
            | dt @ DataType::Date
            | dt @ DataType::Duration(_) => s.cast(dt).unwrap(),
            _ => s,
        };

        df_left.get_columns_mut().insert(join_column_index, s);
        _finish_join(df_left, df_right, suffix.as_deref())
    }
Additional examples can be found in:

Get a reference to the mapping of categorical types to the string values.

Examples found in repository?
src/chunked_array/logical/categorical/merge.rs (line 82)
81
82
83

    pub(crate) fn merge_categorical_map(&self, other: &Self) -> PolarsResult<Arc<RevMapping>> {
        merge_categorical_map(self.get_rev_map(), other.get_rev_map())
    }
More examples
Hide additional examples
src/chunked_array/logical/categorical/mod.rs (line 129)
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187

    pub fn iter_str(&self) -> CatIter<'_> {
        let iter = self.logical().into_iter();
        CatIter {
            rev: self.get_rev_map(),
            iter,
        }
    }
}

impl LogicalType for CategoricalChunked {
    fn dtype(&self) -> &DataType {
        self.logical.2.as_ref().unwrap()
    }

    fn get_any_value(&self, i: usize) -> PolarsResult<AnyValue<'_>> {
        if i < self.len() {
            Ok(unsafe { self.get_any_value_unchecked(i) })
        } else {
            Err(PolarsError::ComputeError("Index is out of bounds.".into()))
        }
    }

    unsafe fn get_any_value_unchecked(&self, i: usize) -> AnyValue<'_> {
        match self.logical.0.get_unchecked(i) {
            Some(i) => AnyValue::Categorical(i, self.get_rev_map()),
            None => AnyValue::Null,
        }
    }

    fn cast(&self, dtype: &DataType) -> PolarsResult<Series> {
        match dtype {
            DataType::Utf8 => {
                let mapping = &**self.get_rev_map();

                let mut builder =
                    Utf8ChunkedBuilder::new(self.logical.name(), self.len(), self.len() * 5);

                let f = |idx: u32| mapping.get(idx);

                if !self.logical.has_validity() {
                    self.logical
                        .into_no_null_iter()
                        .for_each(|idx| builder.append_value(f(idx)));
                } else {
                    self.logical.into_iter().for_each(|opt_idx| {
                        builder.append_option(opt_idx.map(f));
                    });
                }

                let ca = builder.finish();
                Ok(ca.into_series())
            }
            DataType::UInt32 => {
                let ca =
                    UInt32Chunked::from_chunks(self.logical.name(), self.logical.chunks.clone());
                Ok(ca.into_series())
            }
            #[cfg(feature = "dtype-categorical")]
            DataType::Categorical(_) => Ok(self.clone().into_series()),
            _ => self.logical.cast(dtype),
        }
    }
src/chunked_array/ops/compare_inner.rs (line 314)
313
314
315
316
317
318

    fn into_partial_ord_inner(self) -> Box<dyn PartialOrdInner + 'a> {
        match &**self.get_rev_map() {
            RevMapping::Local(_) => Box::new(CategoricalTakeRandomLocal::new(self)),
            RevMapping::Global(_, _, _) => Box::new(CategoricalTakeRandomGlobal::new(self)),
        }
    }
src/chunked_array/logical/categorical/ops/take_random.rs (line 26)
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

    pub(crate) fn new(ca: &'a CategoricalChunked) -> Self {
        // should be rechunked upstream
        assert_eq!(ca.logical.chunks.len(), 1, "implementation error");
        if let RevMapping::Local(rev_map) = &**ca.get_rev_map() {
            let cats = ca.logical().take_rand();
            Self { rev_map, cats }
        } else {
            unreachable!()
        }
    }
}

impl PartialOrdInner for CategoricalTakeRandomLocal<'_> {
    unsafe fn cmp_element_unchecked(&self, idx_a: usize, idx_b: usize) -> Ordering {
        let a = self
            .cats
            .get_unchecked(idx_a)
            .map(|cat| self.rev_map.value_unchecked(cat as usize));
        let b = self
            .cats
            .get_unchecked(idx_b)
            .map(|cat| self.rev_map.value_unchecked(cat as usize));
        a.partial_cmp(&b).unwrap()
    }
}

pub(crate) struct CategoricalTakeRandomGlobal<'a> {
    rev_map_part_1: &'a PlHashMap<u32, u32>,
    rev_map_part_2: &'a Utf8Array<i64>,
    cats: TakeCats<'a>,
}
impl<'a> CategoricalTakeRandomGlobal<'a> {
    pub(crate) fn new(ca: &'a CategoricalChunked) -> Self {
        // should be rechunked upstream
        assert_eq!(ca.logical.chunks.len(), 1, "implementation error");
        if let RevMapping::Global(rev_map_part_1, rev_map_part_2, _) = &**ca.get_rev_map() {
            let cats = ca.logical().take_rand();
            Self {
                rev_map_part_1,
                rev_map_part_2,
                cats,
            }
        } else {
            unreachable!()
        }
    }
src/series/implementations/categorical.rs (line 28)
26
27
28
29
30
31
32
33
34
35

    fn finish_with_state(&self, keep_fast_unique: bool, cats: UInt32Chunked) -> CategoricalChunked {
        let mut out = unsafe {
            CategoricalChunked::from_cats_and_rev_map_unchecked(cats, self.0.get_rev_map().clone())
        };
        if keep_fast_unique && self.0.can_fast_unique() {
            out.set_fast_unique(true)
        }
        out.set_lexical_sorted(self.0.use_lexical_sort());
        out
    }
src/series/comparison.rs (line 75)
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

fn compare_cat_to_str_value<Compare>(
    cat: &Series,
    value: &str,
    name: &str,
    compare: Compare,
    fill_value: bool,
) -> PolarsResult<BooleanChunked>
where
    Compare: Fn(&Series, u32) -> PolarsResult<BooleanChunked>,
{
    let cat = cat.categorical().expect("should be categorical");
    let cat_map = cat.get_rev_map();
    match cat_map.find(value) {
        None => Ok(BooleanChunked::full(name, fill_value, cat.len())),
        Some(cat_idx) => {
            let cat = cat.cast(&DataType::UInt32).unwrap();
            compare(&cat, cat_idx)
        }
    }
}
Additional examples can be found in:

Create an [Iterator] that iterates over the &str values of the [CategoricalChunked].

Examples found in repository?
src/chunked_array/ops/sort/categorical.rs (line 57)
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139

    pub fn sort_with(&self, options: SortOptions) -> CategoricalChunked {
        assert!(
            !options.nulls_last,
            "null last not yet supported for categorical dtype"
        );

        if self.use_lexical_sort() {
            match &**self.get_rev_map() {
                RevMapping::Local(arr) => {
                    // we don't use arrow2 sort here because its not activated
                    // that saves compilation
                    let ca = Utf8Chunked::from_chunks("", vec![Box::from(arr.clone())]);
                    let sorted = ca.sort(options.descending);
                    let arr = sorted.downcast_iter().next().unwrap().clone();
                    let rev_map = RevMapping::Local(arr);
                    // safety:
                    // we only reordered the indexes so we are still in bounds
                    unsafe {
                        CategoricalChunked::from_cats_and_rev_map_unchecked(
                            self.logical().clone(),
                            Arc::new(rev_map),
                        )
                    }
                }
                RevMapping::Global(_, _, _) => {
                    // a global rev map must always point to the same string values
                    // so we cannot sort the categories.

                    let mut vals = self
                        .logical()
                        .into_no_null_iter()
                        .zip(self.iter_str())
                        .collect_trusted::<Vec<_>>();

                    argsort_branch(
                        vals.as_mut_slice(),
                        options.descending,
                        |(_, a), (_, b)| order_default_null(a, b),
                        |(_, a), (_, b)| order_reverse_null(a, b),
                    );
                    let cats: NoNull<UInt32Chunked> =
                        vals.into_iter().map(|(idx, _v)| idx).collect_trusted();
                    // safety:
                    // we only reordered the indexes so we are still in bounds
                    unsafe {
                        CategoricalChunked::from_cats_and_rev_map_unchecked(
                            cats.into_inner(),
                            self.get_rev_map().clone(),
                        )
                    }
                }
            }
        } else {
            let cats = self.logical().sort_with(options);
            // safety:
            // we only reordered the indexes so we are still in bounds
            unsafe {
                CategoricalChunked::from_cats_and_rev_map_unchecked(
                    cats,
                    self.get_rev_map().clone(),
                )
            }
        }
    }

    /// Returned a sorted `ChunkedArray`.
    #[must_use]
    pub fn sort(&self, reverse: bool) -> CategoricalChunked {
        self.sort_with(SortOptions {
            nulls_last: false,
            descending: reverse,
        })
    }

    /// Retrieve the indexes needed to sort this array.
    pub fn argsort(&self, options: SortOptions) -> IdxCa {
        if self.use_lexical_sort() {
            let iters = [self.iter_str()];
            argsort::argsort(
                self.name(),
                iters,
                options,
                self.logical().null_count(),
                self.len(),
            )
        } else {
            self.logical().argsort(options)
        }
    }

    /// Retrieve the indexes need to sort this and the other arrays.
    #[cfg(feature = "sort_multiple")]
    pub(crate) fn argsort_multiple(
        &self,
        other: &[Series],
        reverse: &[bool],
    ) -> PolarsResult<IdxCa> {
        if self.use_lexical_sort() {
            args_validate(self.logical(), other, reverse)?;
            let mut count: IdxSize = 0;
            let vals: Vec<_> = self
                .iter_str()
                .map(|v| {
                    let i = count;
                    count += 1;
                    (i, v)
                })
                .collect_trusted();

            argsort_multiple_impl(vals, other, reverse)
        } else {
            self.logical().argsort_multiple(other, reverse)
        }
    }

Trait Implementations§

Returns a copy of the value. Read more
Performs copy-assignment from source. Read more
Converts to this type from the input type.
Converts to this type from the input type.
Get data type of ChunkedArray.
Gets AnyValue from LogicalType
Safety Read more

Auto Trait Implementations§

Blanket Implementations§

Gets the TypeId of self. Read more
Immutably borrows from an owned value. Read more
Mutably borrows from an owned value. Read more
source§

fn __clone_box(&self, _: Private) -> *mut ()

Returns the argument unchanged.

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

The alignment of pointer.
The type for initializers.
Initializes a with the given initializer. Read more
Dereferences the given pointer. Read more
Mutably dereferences the given pointer. Read more
Drops the object pointed to by the given pointer. Read more
The resulting type after obtaining ownership.
Creates owned data from borrowed data, usually by cloning. Read more
Uses borrowed data to replace owned data, usually by cloning. Read more
The type returned in the event of a conversion error.
Performs the conversion.
The type returned in the event of a conversion error.
Performs the conversion.