datafusion_common/unnest.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! [`UnnestOptions`] for unnesting structured types
19
20use crate::Column;
21
22/// How [`UnnestOptions`] handles `NULL` and empty list values in the input column.
23///
24/// The variants enumerate the three observable behaviors so that callers do
25/// not have to compose multiple boolean flags to express what they want.
26#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Hash)]
27pub enum NullHandling {
28 /// Drop rows where the input list is `NULL` or empty. Matches the
29 /// default behavior of systems such as DuckDB and ClickHouse.
30 Drop,
31 /// Preserve `NULL` input rows as a single output row containing `NULL`.
32 /// Empty lists still produce zero output rows. This is the default and
33 /// matches DataFusion's historical `preserve_nulls = true` behavior.
34 #[default]
35 Preserve,
36 /// Like [`Self::Preserve`], and additionally treat an empty list
37 /// identically to a `NULL` list, producing a single output row
38 /// containing `NULL`.
39 PreserveAndExpandEmpty,
40}
41
42/// Options for unnesting a column that contains a list type,
43/// replicating values in the other, non nested rows.
44///
45/// Conceptually this operation is like joining each row with all the
46/// values in the list column.
47///
48/// The behavior with `NULL` and empty input lists is controlled by
49/// [`NullHandling`]. See its variants for full details.
50///
51/// # Examples
52///
53/// ## `Unnest(c1)`, null_handling: NullHandling::Drop
54/// ```text
55/// ┌─────────┐ ┌─────┐ ┌─────────┐ ┌─────┐
56/// │ {1, 2} │ │ A │ Unnest │ 1 │ │ A │
57/// ├─────────┤ ├─────┤ ├─────────┤ ├─────┤
58/// │ null │ │ B │ │ 2 │ │ A │
59/// ├─────────┤ ├─────┤ ────────────▶ ├─────────┤ ├─────┤
60/// │ {} │ │ D │ │ 3 │ │ E │
61/// ├─────────┤ ├─────┤ └─────────┘ └─────┘
62/// │ {3} │ │ E │ c1 c2
63/// └─────────┘ └─────┘
64/// c1 c2
65/// ```
66///
67/// ## `Unnest(c1)`, null_handling: NullHandling::Preserve
68/// ```text
69/// ┌─────────┐ ┌─────┐ ┌─────────┐ ┌─────┐
70/// │ {1, 2} │ │ A │ Unnest │ 1 │ │ A │
71/// ├─────────┤ ├─────┤ ├─────────┤ ├─────┤
72/// │ null │ │ B │ │ 2 │ │ A │
73/// ├─────────┤ ├─────┤ ────────────▶ ├─────────┤ ├─────┤
74/// │ {} │ │ D │ │ null │ │ B │
75/// ├─────────┤ ├─────┤ ├─────────┤ ├─────┤
76/// │ {3} │ │ E │ │ 3 │ │ E │
77/// └─────────┘ └─────┘ └─────────┘ └─────┘
78/// c1 c2 c1 c2
79/// ```
80///
81/// ## `Unnest(c1)`, null_handling: NullHandling::PreserveAndExpandEmpty
82/// ```text
83/// ┌─────────┐ ┌─────┐ ┌─────────┐ ┌─────┐
84/// │ {1, 2} │ │ A │ Unnest │ 1 │ │ A │
85/// ├─────────┤ ├─────┤ ├─────────┤ ├─────┤
86/// │ null │ │ B │ │ 2 │ │ A │
87/// ├─────────┤ ├─────┤ ────────────▶ ├─────────┤ ├─────┤
88/// │ {} │ │ D │ │ null │ │ B │
89/// ├─────────┤ ├─────┤ ├─────────┤ ├─────┤
90/// │ {3} │ │ E │ │ null │ │ D │
91/// └─────────┘ └─────┘ ├─────────┤ ├─────┤
92/// c1 c2 │ 3 │ │ E │
93/// └─────────┘ └─────┘
94/// c1 c2
95/// ```
96///
97/// `recursions` instruct how a column should be unnested (e.g unnesting a column multiple
98/// time, with depth = 1 and depth = 2). Any unnested column not being mentioned inside this
99/// options is inferred to be unnested with depth = 1
100#[derive(Debug, Clone, PartialEq, PartialOrd, Hash, Eq)]
101pub struct UnnestOptions {
102 /// How to handle `NULL` and empty list values in the input column.
103 /// Defaults to [`NullHandling::Preserve`].
104 pub null_handling: NullHandling,
105 /// If specific columns need to be unnested multiple times (e.g at different depth),
106 /// declare them here. Any unnested columns not being mentioned inside this option
107 /// will be unnested with depth = 1
108 pub recursions: Vec<RecursionUnnestOption>,
109}
110
111/// Instruction on how to unnest a column (mostly with a list type)
112/// such as how to name the output, and how many level it should be unnested
113#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)]
114pub struct RecursionUnnestOption {
115 pub input_column: Column,
116 pub output_column: Column,
117 pub depth: usize,
118}
119
120impl Default for UnnestOptions {
121 fn default() -> Self {
122 Self {
123 null_handling: NullHandling::Preserve,
124 recursions: vec![],
125 }
126 }
127}
128
129impl UnnestOptions {
130 /// Create a new [`UnnestOptions`] with default values
131 pub fn new() -> Self {
132 Default::default()
133 }
134
135 /// Set the [`NullHandling`] mode used when unnesting `NULL` or empty
136 /// input lists.
137 pub fn with_null_handling(mut self, null_handling: NullHandling) -> Self {
138 self.null_handling = null_handling;
139 self
140 }
141
142 /// Backward-compatible setter that maps the previous boolean
143 /// `preserve_nulls` flag onto [`NullHandling`].
144 ///
145 /// `true` maps to [`NullHandling::Preserve`]; `false` maps to
146 /// [`NullHandling::Drop`]. To opt into the new empty-list-preserving
147 /// mode, call [`Self::with_null_handling`] directly with
148 /// [`NullHandling::PreserveAndExpandEmpty`].
149 pub fn with_preserve_nulls(self, preserve_nulls: bool) -> Self {
150 let null_handling = if preserve_nulls {
151 NullHandling::Preserve
152 } else {
153 NullHandling::Drop
154 };
155 self.with_null_handling(null_handling)
156 }
157
158 /// Returns true if `NULL` input rows produce a single output row
159 /// containing `NULL`.
160 pub fn preserve_nulls(&self) -> bool {
161 !matches!(self.null_handling, NullHandling::Drop)
162 }
163
164 /// Returns true if empty input lists should produce a single
165 /// output row containing `NULL`.
166 pub fn expand_empty_as_null(&self) -> bool {
167 matches!(self.null_handling, NullHandling::PreserveAndExpandEmpty)
168 }
169
170 /// Set the recursions for the unnest operation
171 pub fn with_recursions(mut self, recursion: RecursionUnnestOption) -> Self {
172 self.recursions.push(recursion);
173 self
174 }
175}